Branch data Line data Source code
1 : : /* Debuginfo-over-http server.
2 : : Copyright (C) 2019-2021 Red Hat, Inc.
3 : : This file is part of elfutils.
4 : :
5 : : This file is free software; you can redistribute it and/or modify
6 : : it under the terms of the GNU General Public License as published by
7 : : the Free Software Foundation; either version 3 of the License, or
8 : : (at your option) any later version.
9 : :
10 : : elfutils is distributed in the hope that it will be useful, but
11 : : WITHOUT ANY WARRANTY; without even the implied warranty of
12 : : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : : GNU General Public License for more details.
14 : :
15 : : You should have received a copy of the GNU General Public License
16 : : along with this program. If not, see <http://www.gnu.org/licenses/>. */
17 : :
18 : :
19 : : /* cargo-cult from libdwfl linux-kernel-modules.c */
20 : : /* In case we have a bad fts we include this before config.h because it
21 : : can't handle _FILE_OFFSET_BITS.
22 : : Everything we need here is fine if its declarations just come first.
23 : : Also, include sys/types.h before fts. On some systems fts.h is not self
24 : : contained. */
25 : : #ifdef BAD_FTS
26 : : #include <sys/types.h>
27 : : #include <fts.h>
28 : : #endif
29 : :
30 : : #ifdef HAVE_CONFIG_H
31 : : #include "config.h"
32 : : #endif
33 : :
34 : : extern "C" {
35 : : #include "printversion.h"
36 : : }
37 : :
38 : : #include "debuginfod.h"
39 : : #include <dwarf.h>
40 : :
41 : : #include <argp.h>
42 : : #ifdef __GNUC__
43 : : #undef __attribute__ /* glibc bug - rhbz 1763325 */
44 : : #endif
45 : :
46 : : #include <unistd.h>
47 : : #include <stdlib.h>
48 : : #include <error.h>
49 : : #include <libintl.h>
50 : : #include <locale.h>
51 : : #include <pthread.h>
52 : : #include <signal.h>
53 : : #include <sys/stat.h>
54 : : #include <sys/time.h>
55 : : #include <sys/vfs.h>
56 : : #include <unistd.h>
57 : : #include <fcntl.h>
58 : : #include <netdb.h>
59 : :
60 : :
61 : : /* If fts.h is included before config.h, its indirect inclusions may not
62 : : give us the right LFS aliases of these functions, so map them manually. */
63 : : #ifdef BAD_FTS
64 : : #ifdef _FILE_OFFSET_BITS
65 : : #define open open64
66 : : #define fopen fopen64
67 : : #endif
68 : : #else
69 : : #include <sys/types.h>
70 : : #include <fts.h>
71 : : #endif
72 : :
73 : : #include <cstring>
74 : : #include <vector>
75 : : #include <set>
76 : : #include <map>
77 : : #include <string>
78 : : #include <iostream>
79 : : #include <iomanip>
80 : : #include <ostream>
81 : : #include <sstream>
82 : : #include <mutex>
83 : : #include <deque>
84 : : #include <condition_variable>
85 : : #include <thread>
86 : : // #include <regex> // on rhel7 gcc 4.8, not competent
87 : : #include <regex.h>
88 : : // #include <algorithm>
89 : : using namespace std;
90 : :
91 : : #include <gelf.h>
92 : : #include <libdwelf.h>
93 : :
94 : : #include <microhttpd.h>
95 : :
96 : : #if MHD_VERSION >= 0x00097002
97 : : // libmicrohttpd 0.9.71 broke API
98 : : #define MHD_RESULT enum MHD_Result
99 : : #else
100 : : #define MHD_RESULT int
101 : : #endif
102 : :
103 : : #include <curl/curl.h>
104 : : #include <archive.h>
105 : : #include <archive_entry.h>
106 : : #include <sqlite3.h>
107 : :
108 : : #ifdef __linux__
109 : : #include <sys/syscall.h>
110 : : #endif
111 : :
112 : : #ifdef __linux__
113 : : #define tid() syscall(SYS_gettid)
114 : : #else
115 : : #define tid() pthread_self()
116 : : #endif
117 : :
118 : :
119 : : inline bool
120 : 729 : string_endswith(const string& haystack, const string& needle)
121 : : {
122 [ + - ]: 1458 : return (haystack.size() >= needle.size() &&
123 [ + - + + ]: 729 : equal(haystack.end()-needle.size(), haystack.end(),
124 : 729 : needle.begin()));
125 : : }
126 : :
127 : :
128 : : // Roll this identifier for every sqlite schema incompatibility.
129 : : #define BUILDIDS "buildids9"
130 : :
131 : : #if SQLITE_VERSION_NUMBER >= 3008000
132 : : #define WITHOUT_ROWID "without rowid"
133 : : #else
134 : : #define WITHOUT_ROWID ""
135 : : #endif
136 : :
137 : : static const char DEBUGINFOD_SQLITE_DDL[] =
138 : : "pragma foreign_keys = on;\n"
139 : : "pragma synchronous = 0;\n" // disable fsync()s - this cache is disposable across a machine crash
140 : : "pragma journal_mode = wal;\n" // https://sqlite.org/wal.html
141 : : "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file
142 : : "pragma journal_size_limit = 0;\n" // limit steady state file (between grooming, which also =truncate's)
143 : : "pragma auto_vacuum = incremental;\n" // https://sqlite.org/pragma.html
144 : : "pragma busy_timeout = 1000;\n" // https://sqlite.org/pragma.html
145 : : // NB: all these are overridable with -D option
146 : :
147 : : // Normalization table for interning file names
148 : : "create table if not exists " BUILDIDS "_files (\n"
149 : : " id integer primary key not null,\n"
150 : : " name text unique not null\n"
151 : : " );\n"
152 : : // Normalization table for interning buildids
153 : : "create table if not exists " BUILDIDS "_buildids (\n"
154 : : " id integer primary key not null,\n"
155 : : " hex text unique not null);\n"
156 : : // Track the completion of scanning of a given file & sourcetype at given time
157 : : "create table if not exists " BUILDIDS "_file_mtime_scanned (\n"
158 : : " mtime integer not null,\n"
159 : : " file integer not null,\n"
160 : : " size integer not null,\n" // in bytes
161 : : " sourcetype text(1) not null\n"
162 : : " check (sourcetype IN ('F', 'R')),\n"
163 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
164 : : " primary key (file, mtime, sourcetype)\n"
165 : : " ) " WITHOUT_ROWID ";\n"
166 : : "create table if not exists " BUILDIDS "_f_de (\n"
167 : : " buildid integer not null,\n"
168 : : " debuginfo_p integer not null,\n"
169 : : " executable_p integer not null,\n"
170 : : " file integer not null,\n"
171 : : " mtime integer not null,\n"
172 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
173 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
174 : : " primary key (buildid, file, mtime)\n"
175 : : " ) " WITHOUT_ROWID ";\n"
176 : : "create table if not exists " BUILDIDS "_f_s (\n"
177 : : " buildid integer not null,\n"
178 : : " artifactsrc integer not null,\n"
179 : : " file integer not null,\n" // NB: not necessarily entered into _mtime_scanned
180 : : " mtime integer not null,\n"
181 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
182 : : " foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
183 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
184 : : " primary key (buildid, artifactsrc, file, mtime)\n"
185 : : " ) " WITHOUT_ROWID ";\n"
186 : : "create table if not exists " BUILDIDS "_r_de (\n"
187 : : " buildid integer not null,\n"
188 : : " debuginfo_p integer not null,\n"
189 : : " executable_p integer not null,\n"
190 : : " file integer not null,\n"
191 : : " mtime integer not null,\n"
192 : : " content integer not null,\n"
193 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
194 : : " foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
195 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
196 : : " primary key (buildid, debuginfo_p, executable_p, file, content, mtime)\n"
197 : : " ) " WITHOUT_ROWID ";\n"
198 : : "create table if not exists " BUILDIDS "_r_sref (\n" // outgoing dwarf sourcefile references from rpm
199 : : " buildid integer not null,\n"
200 : : " artifactsrc integer not null,\n"
201 : : " foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
202 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
203 : : " primary key (buildid, artifactsrc)\n"
204 : : " ) " WITHOUT_ROWID ";\n"
205 : : "create table if not exists " BUILDIDS "_r_sdef (\n" // rpm contents that may satisfy sref
206 : : " file integer not null,\n"
207 : : " mtime integer not null,\n"
208 : : " content integer not null,\n"
209 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
210 : : " foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
211 : : " primary key (content, file, mtime)\n"
212 : : " ) " WITHOUT_ROWID ";\n"
213 : : // create views to glue together some of the above tables, for webapi D queries
214 : : "create view if not exists " BUILDIDS "_query_d as \n"
215 : : "select\n"
216 : : " b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"
217 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_f_de n\n"
218 : : " where b.id = n.buildid and f0.id = n.file and n.debuginfo_p = 1\n"
219 : : "union all select\n"
220 : : " b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"
221 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_r_de n\n"
222 : : " where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.debuginfo_p = 1\n"
223 : : ";"
224 : : // ... and for E queries
225 : : "create view if not exists " BUILDIDS "_query_e as \n"
226 : : "select\n"
227 : : " b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"
228 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_f_de n\n"
229 : : " where b.id = n.buildid and f0.id = n.file and n.executable_p = 1\n"
230 : : "union all select\n"
231 : : " b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"
232 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_r_de n\n"
233 : : " where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.executable_p = 1\n"
234 : : ";"
235 : : // ... and for S queries
236 : : "create view if not exists " BUILDIDS "_query_s as \n"
237 : : "select\n"
238 : : " b.hex as buildid, fs.name as artifactsrc, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1, null as source0ref\n"
239 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files fs, " BUILDIDS "_f_s n\n"
240 : : " where b.id = n.buildid and f0.id = n.file and fs.id = n.artifactsrc\n"
241 : : "union all select\n"
242 : : " b.hex as buildid, f1.name as artifactsrc, 'R' as sourcetype, f0.name as source0, sd.mtime as mtime, f1.name as source1, fsref.name as source0ref\n"
243 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_files fsref, "
244 : : " " BUILDIDS "_r_sdef sd, " BUILDIDS "_r_sref sr, " BUILDIDS "_r_de sde\n"
245 : : " where b.id = sr.buildid and f0.id = sd.file and fsref.id = sde.file and f1.id = sd.content\n"
246 : : " and sr.artifactsrc = sd.content and sde.buildid = sr.buildid\n"
247 : : ";"
248 : : // and for startup overview counts
249 : : "drop view if exists " BUILDIDS "_stats;\n"
250 : : "create view if not exists " BUILDIDS "_stats as\n"
251 : : " select 'file d/e' as label,count(*) as quantity from " BUILDIDS "_f_de\n"
252 : : "union all select 'file s',count(*) from " BUILDIDS "_f_s\n"
253 : : "union all select 'archive d/e',count(*) from " BUILDIDS "_r_de\n"
254 : : "union all select 'archive sref',count(*) from " BUILDIDS "_r_sref\n"
255 : : "union all select 'archive sdef',count(*) from " BUILDIDS "_r_sdef\n"
256 : : "union all select 'buildids',count(*) from " BUILDIDS "_buildids\n"
257 : : "union all select 'filenames',count(*) from " BUILDIDS "_files\n"
258 : : "union all select 'files scanned (#)',count(*) from " BUILDIDS "_file_mtime_scanned\n"
259 : : "union all select 'files scanned (mb)',coalesce(sum(size)/1024/1024,0) from " BUILDIDS "_file_mtime_scanned\n"
260 : : #if SQLITE_VERSION_NUMBER >= 3016000
261 : : "union all select 'index db size (mb)',page_count*page_size/1024/1024 as size FROM pragma_page_count(), pragma_page_size()\n"
262 : : #endif
263 : : ";\n"
264 : :
265 : : // schema change history & garbage collection
266 : : //
267 : : // XXX: we could have migration queries here to bring prior-schema
268 : : // data over instead of just dropping it.
269 : : //
270 : : // buildids9: widen the mtime_scanned table
271 : : "" // <<< we are here
272 : : // buildids8: slim the sref table
273 : : "drop table if exists buildids8_f_de;\n"
274 : : "drop table if exists buildids8_f_s;\n"
275 : : "drop table if exists buildids8_r_de;\n"
276 : : "drop table if exists buildids8_r_sref;\n"
277 : : "drop table if exists buildids8_r_sdef;\n"
278 : : "drop table if exists buildids8_file_mtime_scanned;\n"
279 : : "drop table if exists buildids8_files;\n"
280 : : "drop table if exists buildids8_buildids;\n"
281 : : // buildids7: separate _norm table into dense subtype tables
282 : : "drop table if exists buildids7_f_de;\n"
283 : : "drop table if exists buildids7_f_s;\n"
284 : : "drop table if exists buildids7_r_de;\n"
285 : : "drop table if exists buildids7_r_sref;\n"
286 : : "drop table if exists buildids7_r_sdef;\n"
287 : : "drop table if exists buildids7_file_mtime_scanned;\n"
288 : : "drop table if exists buildids7_files;\n"
289 : : "drop table if exists buildids7_buildids;\n"
290 : : // buildids6: drop bolo/rfolo again, represent sources / rpmcontents in main table
291 : : "drop table if exists buildids6_norm;\n"
292 : : "drop table if exists buildids6_files;\n"
293 : : "drop table if exists buildids6_buildids;\n"
294 : : "drop view if exists buildids6;\n"
295 : : // buildids5: redefine srcfile1 column to be '.'-less (for rpms)
296 : : "drop table if exists buildids5_norm;\n"
297 : : "drop table if exists buildids5_files;\n"
298 : : "drop table if exists buildids5_buildids;\n"
299 : : "drop table if exists buildids5_bolo;\n"
300 : : "drop table if exists buildids5_rfolo;\n"
301 : : "drop view if exists buildids5;\n"
302 : : // buildids4: introduce rpmfile RFOLO
303 : : "drop table if exists buildids4_norm;\n"
304 : : "drop table if exists buildids4_files;\n"
305 : : "drop table if exists buildids4_buildids;\n"
306 : : "drop table if exists buildids4_bolo;\n"
307 : : "drop table if exists buildids4_rfolo;\n"
308 : : "drop view if exists buildids4;\n"
309 : : // buildids3*: split out srcfile BOLO
310 : : "drop table if exists buildids3_norm;\n"
311 : : "drop table if exists buildids3_files;\n"
312 : : "drop table if exists buildids3_buildids;\n"
313 : : "drop table if exists buildids3_bolo;\n"
314 : : "drop view if exists buildids3;\n"
315 : : // buildids2: normalized buildid and filenames into interning tables;
316 : : "drop table if exists buildids2_norm;\n"
317 : : "drop table if exists buildids2_files;\n"
318 : : "drop table if exists buildids2_buildids;\n"
319 : : "drop view if exists buildids2;\n"
320 : : // buildids1: made buildid and artifacttype NULLable, to represent cached-negative
321 : : // lookups from sources, e.g. files or rpms that contain no buildid-indexable content
322 : : "drop table if exists buildids1;\n"
323 : : // buildids: original
324 : : "drop table if exists buildids;\n"
325 : : ;
326 : :
327 : : static const char DEBUGINFOD_SQLITE_CLEANUP_DDL[] =
328 : : "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file
329 : : ;
330 : :
331 : :
332 : :
333 : :
334 : : /* Name and version of program. */
335 : : /* ARGP_PROGRAM_VERSION_HOOK_DEF = print_version; */ // not this simple for C++
336 : :
337 : : /* Bug report address. */
338 : : ARGP_PROGRAM_BUG_ADDRESS_DEF = PACKAGE_BUGREPORT;
339 : :
340 : : /* Definitions of arguments for argp functions. */
341 : : static const struct argp_option options[] =
342 : : {
343 : : { NULL, 0, NULL, 0, "Scanners:", 1 },
344 : : { "scan-file-dir", 'F', NULL, 0, "Enable ELF/DWARF file scanning.", 0 },
345 : : { "scan-rpm-dir", 'R', NULL, 0, "Enable RPM scanning.", 0 },
346 : : { "scan-deb-dir", 'U', NULL, 0, "Enable DEB scanning.", 0 },
347 : : { "scan-archive", 'Z', "EXT=CMD", 0, "Enable arbitrary archive scanning.", 0 },
348 : : // "source-oci-imageregistry" ...
349 : :
350 : : { NULL, 0, NULL, 0, "Options:", 2 },
351 : : { "logical", 'L', NULL, 0, "Follow symlinks, default=ignore.", 0 },
352 : : { "rescan-time", 't', "SECONDS", 0, "Number of seconds to wait between rescans, 0=disable.", 0 },
353 : : { "groom-time", 'g', "SECONDS", 0, "Number of seconds to wait between database grooming, 0=disable.", 0 },
354 : : { "maxigroom", 'G', NULL, 0, "Run a complete database groom/shrink pass at startup.", 0 },
355 : : { "concurrency", 'c', "NUM", 0, "Limit scanning thread concurrency to NUM.", 0 },
356 : : { "include", 'I', "REGEX", 0, "Include files matching REGEX, default=all.", 0 },
357 : : { "exclude", 'X', "REGEX", 0, "Exclude files matching REGEX, default=none.", 0 },
358 : : { "port", 'p', "NUM", 0, "HTTP port to listen on, default 8002.", 0 },
359 : : { "database", 'd', "FILE", 0, "Path to sqlite database.", 0 },
360 : : { "ddl", 'D', "SQL", 0, "Apply extra sqlite ddl/pragma to connection.", 0 },
361 : : { "verbose", 'v', NULL, 0, "Increase verbosity.", 0 },
362 : : #define ARGP_KEY_FDCACHE_FDS 0x1001
363 : : { "fdcache-fds", ARGP_KEY_FDCACHE_FDS, "NUM", 0, "Maximum number of archive files to keep in fdcache.", 0 },
364 : : #define ARGP_KEY_FDCACHE_MBS 0x1002
365 : : { "fdcache-mbs", ARGP_KEY_FDCACHE_MBS, "MB", 0, "Maximum total size of archive file fdcache.", 0 },
366 : : #define ARGP_KEY_FDCACHE_PREFETCH 0x1003
367 : : { "fdcache-prefetch", ARGP_KEY_FDCACHE_PREFETCH, "NUM", 0, "Number of archive files to prefetch into fdcache.", 0 },
368 : : #define ARGP_KEY_FDCACHE_MINTMP 0x1004
369 : : { "fdcache-mintmp", ARGP_KEY_FDCACHE_MINTMP, "NUM", 0, "Minimum free space% on tmpdir.", 0 },
370 : : { NULL, 0, NULL, 0, NULL, 0 }
371 : : };
372 : :
373 : : /* Short description of program. */
374 : : static const char doc[] = "Serve debuginfo-related content across HTTP from files under PATHs.";
375 : :
376 : : /* Strings for arguments in help texts. */
377 : : static const char args_doc[] = "[PATH ...]";
378 : :
379 : : /* Prototype for option handler. */
380 : : static error_t parse_opt (int key, char *arg, struct argp_state *state);
381 : :
382 : : /* Data structure to communicate with argp functions. */
383 : : static struct argp argp =
384 : : {
385 : : options, parse_opt, args_doc, doc, NULL, NULL, NULL
386 : : };
387 : :
388 : :
389 : : static string db_path;
390 : : static sqlite3 *db; // single connection, serialized across all our threads!
391 : : static sqlite3 *dbq; // webapi query-servicing readonly connection, serialized ditto!
392 : : static unsigned verbose;
393 : : static volatile sig_atomic_t interrupted = 0;
394 : : static volatile sig_atomic_t forced_rescan_count = 0;
395 : : static volatile sig_atomic_t sigusr1 = 0;
396 : : static volatile sig_atomic_t forced_groom_count = 0;
397 : : static volatile sig_atomic_t sigusr2 = 0;
398 : : static unsigned http_port = 8002;
399 : : static unsigned rescan_s = 300;
400 : : static unsigned groom_s = 86400;
401 : : static bool maxigroom = false;
402 : : static unsigned concurrency = std::thread::hardware_concurrency() ?: 1;
403 : : static set<string> source_paths;
404 : : static bool scan_files = false;
405 : : static map<string,string> scan_archives;
406 : : static vector<string> extra_ddl;
407 : : static regex_t file_include_regex;
408 : : static regex_t file_exclude_regex;
409 : : static bool traverse_logical;
410 : : static long fdcache_fds;
411 : : static long fdcache_mbs;
412 : : static long fdcache_prefetch;
413 : : static long fdcache_mintmp;
414 : : static string tmpdir;
415 : :
416 : : static void set_metric(const string& key, double value);
417 : : // static void inc_metric(const string& key);
418 : : static void set_metric(const string& metric,
419 : : const string& lname, const string& lvalue,
420 : : double value);
421 : : static void inc_metric(const string& metric,
422 : : const string& lname, const string& lvalue);
423 : : static void add_metric(const string& metric,
424 : : const string& lname, const string& lvalue,
425 : : double value);
426 : : // static void add_metric(const string& metric, double value);
427 : :
428 : : class tmp_inc_metric { // a RAII style wrapper for exception-safe scoped increment & decrement
429 : : string m, n, v;
430 : : public:
431 : 144 : tmp_inc_metric(const string& mname, const string& lname, const string& lvalue):
432 [ + - + - ]: 144 : m(mname), n(lname), v(lvalue)
433 : : {
434 [ + - ]: 144 : add_metric (m, n, v, 1);
435 : 144 : }
436 : 144 : ~tmp_inc_metric()
437 : 144 : {
438 : 144 : add_metric (m, n, v, -1);
439 : 144 : }
440 : : };
441 : :
442 : : class tmp_ms_metric { // a RAII style wrapper for exception-safe scoped timing
443 : : string m, n, v;
444 : : struct timespec ts_start;
445 : : public:
446 : 5136 : tmp_ms_metric(const string& mname, const string& lname, const string& lvalue):
447 [ + - + - ]: 5136 : m(mname), n(lname), v(lvalue)
448 : : {
449 : 5137 : clock_gettime (CLOCK_MONOTONIC, & ts_start);
450 : 5138 : }
451 : 5137 : ~tmp_ms_metric()
452 : 5138 : {
453 : : struct timespec ts_end;
454 : 5137 : clock_gettime (CLOCK_MONOTONIC, & ts_end);
455 : 5137 : double deltas = (ts_end.tv_sec - ts_start.tv_sec)
456 : 5137 : + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
457 : :
458 : 5137 : add_metric (m + "_milliseconds_sum", n, v, (deltas*1000.0));
459 : 5138 : inc_metric (m + "_milliseconds_count", n, v);
460 : 5138 : }
461 : : };
462 : :
463 : :
464 : : /* Handle program arguments. */
465 : : static error_t
466 : 51 : parse_opt (int key, char *arg,
467 : : struct argp_state *state __attribute__ ((unused)))
468 : : {
469 : : int rc;
470 [ + + + + : 51 : switch (key)
+ + + + -
+ + - - -
- + + - +
+ + ]
471 : : {
472 : 10 : case 'v': verbose ++; break;
473 [ + - ]: 3 : case 'd': db_path = string(arg); break;
474 : 3 : case 'p': http_port = (unsigned) atoi(arg);
475 [ + - - + ]: 3 : if (http_port == 0 || http_port > 65535)
476 : 0 : argp_failure(state, 1, EINVAL, "port number");
477 : 3 : break;
478 : 4 : case 'F': scan_files = true; break;
479 : 1 : case 'R':
480 [ + - + - : 1 : scan_archives[".rpm"]="cat"; // libarchive groks rpm natively
+ - ]
481 : 1 : break;
482 : 2 : case 'U':
483 [ + - + - : 2 : scan_archives[".deb"]="(bsdtar -O -x -f - data.tar\\*)<";
+ - ]
484 [ + - + - : 2 : scan_archives[".ddeb"]="(bsdtar -O -x -f - data.tar\\*)<";
+ - ]
485 [ + - + - : 2 : scan_archives[".ipk"]="(bsdtar -O -x -f - data.tar\\*)<";
+ - ]
486 : : // .udeb too?
487 : 2 : break;
488 : 2 : case 'Z':
489 : : {
490 : 2 : char* extension = strchr(arg, '=');
491 [ - + ]: 2 : if (arg[0] == '\0')
492 : 0 : argp_failure(state, 1, EINVAL, "missing EXT");
493 [ + + ]: 2 : else if (extension)
494 [ + - + - : 1 : scan_archives[string(arg, (extension-arg))]=string(extension+1);
+ - ]
495 : : else
496 [ + - + - : 1 : scan_archives[string(arg)]=string("cat");
+ - ]
497 : : }
498 : 2 : break;
499 : 2 : case 'L':
500 : 2 : traverse_logical = true;
501 : 2 : break;
502 [ # # # # ]: 0 : case 'D': extra_ddl.push_back(string(arg)); break;
503 : 1 : case 't':
504 : 1 : rescan_s = (unsigned) atoi(arg);
505 : 1 : break;
506 : 1 : case 'g':
507 : 1 : groom_s = (unsigned) atoi(arg);
508 : 1 : break;
509 : 0 : case 'G':
510 : 0 : maxigroom = true;
511 : 0 : break;
512 : 0 : case 'c':
513 : 0 : concurrency = (unsigned) atoi(arg);
514 [ # # ]: 0 : if (concurrency < 1) concurrency = 1;
515 : 0 : break;
516 : 0 : case 'I':
517 : : // NB: no problem with unconditional free here - an earlier failed regcomp would exit program
518 : 0 : regfree (&file_include_regex);
519 : 0 : rc = regcomp (&file_include_regex, arg, REG_EXTENDED|REG_NOSUB);
520 [ # # ]: 0 : if (rc != 0)
521 : 0 : argp_failure(state, 1, EINVAL, "regular expression");
522 : 0 : break;
523 : 0 : case 'X':
524 : 0 : regfree (&file_exclude_regex);
525 : 0 : rc = regcomp (&file_exclude_regex, arg, REG_EXTENDED|REG_NOSUB);
526 [ # # ]: 0 : if (rc != 0)
527 : 0 : argp_failure(state, 1, EINVAL, "regular expression");
528 : 0 : break;
529 : 1 : case ARGP_KEY_FDCACHE_FDS:
530 : 1 : fdcache_fds = atol (arg);
531 : 1 : break;
532 : 1 : case ARGP_KEY_FDCACHE_MBS:
533 : 1 : fdcache_mbs = atol (arg);
534 : 1 : break;
535 : 0 : case ARGP_KEY_FDCACHE_PREFETCH:
536 : 0 : fdcache_prefetch = atol (arg);
537 : 0 : break;
538 : 1 : case ARGP_KEY_FDCACHE_MINTMP:
539 : 1 : fdcache_mintmp = atol (arg);
540 : 1 : break;
541 : 6 : case ARGP_KEY_ARG:
542 [ + - + - ]: 6 : source_paths.insert(string(arg));
543 : 6 : break;
544 : : // case 'h': argp_state_help (state, stderr, ARGP_HELP_LONG|ARGP_HELP_EXIT_OK);
545 : 13 : default: return ARGP_ERR_UNKNOWN;
546 : : }
547 : :
548 : 38 : return 0;
549 : : }
550 : :
551 : :
552 : : ////////////////////////////////////////////////////////////////////////
553 : :
554 : :
555 : : // represent errors that may get reported to an ostream and/or a libmicrohttpd connection
556 : :
557 : : struct reportable_exception
558 : : {
559 : : int code;
560 : : string message;
561 : :
562 : 8 : reportable_exception(int c, const string& m): code(c), message(m) {}
563 : 36 : reportable_exception(const string& m): code(503), message(m) {}
564 : : reportable_exception(): code(503), message() {}
565 : :
566 : : void report(ostream& o) const; // defined under obatched() class below
567 : :
568 : 12 : MHD_RESULT mhd_send_response(MHD_Connection* c) const {
569 : 24 : MHD_Response* r = MHD_create_response_from_buffer (message.size(),
570 : 12 : (void*) message.c_str(),
571 : : MHD_RESPMEM_MUST_COPY);
572 : 12 : MHD_add_response_header (r, "Content-Type", "text/plain");
573 : 12 : MHD_RESULT rc = MHD_queue_response (c, code, r);
574 : 12 : MHD_destroy_response (r);
575 : 12 : return rc;
576 : : }
577 : : };
578 : :
579 : :
580 : : struct sqlite_exception: public reportable_exception
581 : : {
582 : 2 : sqlite_exception(int rc, const string& msg):
583 [ + - + - : 2 : reportable_exception(string("sqlite3 error: ") + msg + ": " + string(sqlite3_errstr(rc) ?: "?")) {
+ - + - +
- + - + -
+ - ]
584 [ + - + - : 2 : inc_metric("error_count","sqlite3",sqlite3_errstr(rc));
+ - + - +
- ]
585 : 2 : }
586 : : };
587 : :
588 : : struct libc_exception: public reportable_exception
589 : : {
590 : 31 : libc_exception(int rc, const string& msg):
591 [ + - + - : 31 : reportable_exception(string("libc error: ") + msg + ": " + string(strerror(rc) ?: "?")) {
+ - + - +
- + - +
- ]
592 [ + - + - : 31 : inc_metric("error_count","libc",strerror(rc));
+ - + - ]
593 : 31 : }
594 : : };
595 : :
596 : :
597 : : struct archive_exception: public reportable_exception
598 : : {
599 : 0 : archive_exception(const string& msg):
600 [ # # # # : 0 : reportable_exception(string("libarchive error: ") + msg) {
# # ]
601 [ # # # # : 0 : inc_metric("error_count","libarchive",msg);
# # ]
602 : 0 : }
603 : 0 : archive_exception(struct archive* a, const string& msg):
604 [ # # # # : 0 : reportable_exception(string("libarchive error: ") + msg + ": " + string(archive_error_string(a) ?: "?")) {
# # # # #
# # # # #
# # ]
605 [ # # # # : 0 : inc_metric("error_count","libarchive",msg + ": " + string(archive_error_string(a) ?: "?"));
# # # # #
# # # # #
# # ]
606 : 0 : }
607 : : };
608 : :
609 : :
610 : : struct elfutils_exception: public reportable_exception
611 : : {
612 : 0 : elfutils_exception(int rc, const string& msg):
613 [ # # # # : 0 : reportable_exception(string("elfutils error: ") + msg + ": " + string(elf_errmsg(rc) ?: "?")) {
# # # # #
# # # # #
# # ]
614 [ # # # # : 0 : inc_metric("error_count","elfutils",elf_errmsg(rc));
# # # # #
# ]
615 : 0 : }
616 : : };
617 : :
618 : :
619 : : ////////////////////////////////////////////////////////////////////////
620 : :
621 : : template <typename Payload>
622 : : class workq
623 : : {
624 : : set<Payload> q; // eliminate duplicates
625 : : mutex mtx;
626 : : condition_variable cv;
627 : : bool dead;
628 : : unsigned idlers;
629 : :
630 : : public:
631 : 3 : workq() { dead = false; idlers = 0; }
632 : 3 : ~workq() {}
633 : :
634 : 173 : void push_back(const Payload& p)
635 : : {
636 [ + - ]: 346 : unique_lock<mutex> lock(mtx);
637 [ + - ]: 173 : q.insert (p);
638 [ + - + - : 173 : set_metric("thread_work_pending","role","scan", q.size());
+ - + - ]
639 : 173 : cv.notify_all();
640 : 173 : }
641 : :
642 : : // kill this workqueue, wake up all idlers / scanners
643 : 3 : void nuke() {
644 [ + - ]: 6 : unique_lock<mutex> lock(mtx);
645 : : // optional: q.clear();
646 : 3 : dead = true;
647 : 3 : cv.notify_all();
648 : 3 : }
649 : :
650 : : // clear the workqueue, when scanning is interrupted with USR2
651 : 0 : void clear() {
652 [ # # ]: 0 : unique_lock<mutex> lock(mtx);
653 : 0 : q.clear();
654 [ # # # # : 0 : set_metric("thread_work_pending","role","scan", q.size());
# # # # ]
655 : 0 : cv.notify_all(); // maybe wake up waiting idlers
656 : 0 : }
657 : :
658 : : // block this scanner thread until there is work to do and no active
659 : 185 : bool wait_front (Payload& p)
660 : : {
661 [ + - ]: 370 : unique_lock<mutex> lock(mtx);
662 [ + + + + : 687 : while (!dead && (q.size() == 0 || idlers > 0))
+ + + + ]
663 : 502 : cv.wait(lock);
664 [ + + ]: 185 : if (dead)
665 : 12 : return false;
666 : : else
667 : : {
668 [ + - ]: 173 : p = * q.begin();
669 [ + - ]: 173 : q.erase (q.begin());
670 [ + - + - : 173 : set_metric("thread_work_pending","role","scan", q.size());
+ - + - ]
671 [ + + ]: 173 : if (q.size() == 0)
672 : 92 : cv.notify_all(); // maybe wake up waiting idlers
673 : 173 : return true;
674 : : }
675 : : }
676 : :
677 : : // block this idler thread until there is no work to do
678 : 70 : void wait_idle ()
679 : : {
680 [ + - ]: 70 : unique_lock<mutex> lock(mtx);
681 : 70 : cv.notify_all(); // maybe wake up waiting scanners
682 [ + + + + : 73 : while (!dead && (q.size() != 0))
+ + ]
683 : 3 : cv.wait(lock);
684 : 70 : idlers ++;
685 : 70 : }
686 : :
687 : 67 : void done_idle ()
688 : : {
689 [ + - ]: 134 : unique_lock<mutex> lock(mtx);
690 : 67 : idlers --;
691 : 67 : cv.notify_all(); // maybe wake up waiting scanners, but probably not (shutting down)
692 : 67 : }
693 : : };
694 : :
695 : : typedef struct stat stat_t;
696 : : typedef pair<string,stat_t> scan_payload;
697 : 364 : inline bool operator< (const scan_payload& a, const scan_payload& b)
698 : : {
699 : 364 : return a.first < b.first; // don't bother compare the stat fields
700 : : }
701 : : static workq<scan_payload> scanq; // just a single one
702 : : // producer & idler: thread_main_fts_source_paths()
703 : : // consumer: thread_main_scanner()
704 : : // idler: thread_main_groom()
705 : :
706 : :
707 : :
708 : : ////////////////////////////////////////////////////////////////////////
709 : :
710 : :
711 : : // Print a standard timestamp.
712 : : static ostream&
713 : 1734 : timestamp (ostream &o)
714 : : {
715 : : char datebuf[80];
716 : 1734 : char *now2 = NULL;
717 : 1734 : time_t now_t = time(NULL);
718 : 1734 : struct tm *now = gmtime (&now_t);
719 [ + - ]: 1734 : if (now)
720 : : {
721 : 1734 : (void) strftime (datebuf, sizeof (datebuf), "%c", now);
722 : 1734 : now2 = datebuf;
723 : : }
724 : :
725 : : return o << "[" << (now2 ? now2 : "") << "] "
726 [ + - + - : 1734 : << "(" << getpid () << "/" << tid() << "): ";
+ - + - +
- + - + -
+ - + - ]
727 : : }
728 : :
729 : :
730 : : // A little class that impersonates an ostream to the extent that it can
731 : : // take << streaming operations. It batches up the bits into an internal
732 : : // stringstream until it is destroyed; then flushes to the original ostream.
733 : : // It adds a timestamp
734 : : class obatched
735 : : {
736 : : private:
737 : : ostream& o;
738 : : stringstream stro;
739 : : static mutex lock;
740 : : public:
741 : 1734 : obatched(ostream& oo, bool timestamp_p = true): o(oo)
742 : : {
743 [ + - ]: 1734 : if (timestamp_p)
744 [ + - ]: 1734 : timestamp(stro);
745 : 1734 : }
746 : 1734 : ~obatched()
747 : 1734 : {
748 : 3468 : unique_lock<mutex> do_not_cross_the_streams(obatched::lock);
749 : 1734 : o << stro.str();
750 : 1734 : o.flush();
751 : 1734 : }
752 : : operator ostream& () { return stro; }
753 : 1734 : template <typename T> ostream& operator << (const T& t) { stro << t; return stro; }
754 : : };
755 : : mutex obatched::lock; // just the one, since cout/cerr iostreams are not thread-safe
756 : :
757 : :
758 : 43 : void reportable_exception::report(ostream& o) const {
759 [ + - + - ]: 43 : obatched(o) << message << endl;
760 : 43 : }
761 : :
762 : :
763 : : ////////////////////////////////////////////////////////////////////////
764 : :
765 : :
766 : : // RAII style sqlite prepared-statement holder that matches { } block lifetime
767 : :
768 : : struct sqlite_ps
769 : : {
770 : : private:
771 : : sqlite3* db;
772 : : const string nickname;
773 : : const string sql;
774 : : sqlite3_stmt *pp;
775 : :
776 : : sqlite_ps(const sqlite_ps&); // make uncopyable
777 : : sqlite_ps& operator=(const sqlite_ps &); // make unassignable
778 : :
779 : : public:
780 [ + - ]: 278 : sqlite_ps (sqlite3* d, const string& n, const string& s): db(d), nickname(n), sql(s) {
781 : : // tmp_ms_metric tick("sqlite3","prep",nickname);
782 [ - + ]: 278 : if (verbose > 4)
783 [ # # # # : 0 : obatched(clog) << nickname << " prep " << sql << endl;
# # # # #
# ]
784 [ + - ]: 278 : int rc = sqlite3_prepare_v2 (db, sql.c_str(), -1 /* to \0 */, & this->pp, NULL);
785 [ - + ]: 278 : if (rc != SQLITE_OK)
786 [ # # # # ]: 0 : throw sqlite_exception(rc, "prepare " + sql);
787 : 278 : }
788 : :
789 : 2605 : sqlite_ps& reset()
790 : : {
791 [ + - + - : 7814 : tmp_ms_metric tick("sqlite3","reset",nickname);
+ - ]
792 [ + - ]: 2605 : sqlite3_reset(this->pp);
793 : 5209 : return *this;
794 : : }
795 : :
796 : 3356 : sqlite_ps& bind(int parameter, const string& str)
797 : : {
798 [ - + ]: 3356 : if (verbose > 4)
799 [ # # # # : 0 : obatched(clog) << nickname << " bind " << parameter << "=" << str << endl;
# # # # #
# # # ]
800 : 3356 : int rc = sqlite3_bind_text (this->pp, parameter, str.c_str(), -1, SQLITE_TRANSIENT);
801 [ - + ]: 3356 : if (rc != SQLITE_OK)
802 [ # # # # ]: 0 : throw sqlite_exception(rc, "sqlite3 bind");
803 : 3356 : return *this;
804 : : }
805 : :
806 : 1181 : sqlite_ps& bind(int parameter, int64_t value)
807 : : {
808 [ - + ]: 1181 : if (verbose > 4)
809 [ # # # # : 0 : obatched(clog) << nickname << " bind " << parameter << "=" << value << endl;
# # # # #
# # # ]
810 : 1181 : int rc = sqlite3_bind_int64 (this->pp, parameter, value);
811 [ - + ]: 1180 : if (rc != SQLITE_OK)
812 [ # # # # ]: 0 : throw sqlite_exception(rc, "sqlite3 bind");
813 : 1180 : return *this;
814 : : }
815 : :
816 : : sqlite_ps& bind(int parameter)
817 : : {
818 : : if (verbose > 4)
819 : : obatched(clog) << nickname << " bind " << parameter << "=" << "NULL" << endl;
820 : : int rc = sqlite3_bind_null (this->pp, parameter);
821 : : if (rc != SQLITE_OK)
822 : : throw sqlite_exception(rc, "sqlite3 bind");
823 : : return *this;
824 : : }
825 : :
826 : :
827 : 1932 : void step_ok_done() {
828 [ + - + - : 7728 : tmp_ms_metric tick("sqlite3","step_done",nickname);
+ - ]
829 [ + - ]: 1932 : int rc = sqlite3_step (this->pp);
830 [ - + ]: 1932 : if (verbose > 4)
831 [ # # # # : 0 : obatched(clog) << nickname << " step-ok-done(" << sqlite3_errstr(rc) << ") " << sql << endl;
# # # # #
# # # # #
# # ]
832 [ + - + + : 1932 : if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)
+ + ]
833 [ + - + - ]: 1 : throw sqlite_exception(rc, "sqlite3 step");
834 [ + - ]: 1931 : (void) sqlite3_reset (this->pp);
835 : 1931 : }
836 : :
837 : :
838 : 601 : int step() {
839 [ + - + - : 1803 : tmp_ms_metric tick("sqlite3","step",nickname);
+ - ]
840 [ + - ]: 601 : int rc = sqlite3_step (this->pp);
841 [ - + ]: 601 : if (verbose > 4)
842 [ # # # # : 0 : obatched(clog) << nickname << " step(" << sqlite3_errstr(rc) << ") " << sql << endl;
# # # # #
# # # # #
# # ]
843 : 1202 : return rc;
844 : : }
845 : :
846 : 278 : ~sqlite_ps () { sqlite3_finalize (this->pp); }
847 : 1008 : operator sqlite3_stmt* () { return this->pp; }
848 : : };
849 : :
850 : :
851 : : ////////////////////////////////////////////////////////////////////////
852 : :
853 : : // RAII style templated autocloser
854 : :
855 : : template <class Payload, class Ignore>
856 : : struct defer_dtor
857 : : {
858 : : public:
859 : : typedef Ignore (*dtor_fn) (Payload);
860 : :
861 : : private:
862 : : Payload p;
863 : : dtor_fn fn;
864 : :
865 : : public:
866 : 342 : defer_dtor(Payload _p, dtor_fn _fn): p(_p), fn(_fn) {}
867 : 342 : ~defer_dtor() { (void) (*fn)(p); }
868 : :
869 : : private:
870 : : defer_dtor(const defer_dtor<Payload,Ignore>&); // make uncopyable
871 : : defer_dtor& operator=(const defer_dtor<Payload,Ignore> &); // make unassignable
872 : : };
873 : :
874 : :
875 : :
876 : : ////////////////////////////////////////////////////////////////////////
877 : :
878 : :
879 : : static string
880 : 292 : header_censor(const string& str)
881 : : {
882 : 292 : string y;
883 [ + + ]: 3492 : for (auto&& x : str)
884 : : {
885 [ + + + + : 3200 : if (isalnum(x) || x == '/' || x == '.' || x == ',' || x == '_' || x == ':')
+ + + + +
- - + ]
886 [ + - ]: 3199 : y += x;
887 : : }
888 : 292 : return y;
889 : : }
890 : :
891 : :
892 : : static string
893 : 146 : conninfo (struct MHD_Connection * conn)
894 : : {
895 : : char hostname[256]; // RFC1035
896 : : char servname[256];
897 : 146 : int sts = -1;
898 : :
899 [ - + ]: 146 : if (conn == 0)
900 [ # # ]: 0 : return "internal";
901 : :
902 : : /* Look up client address data. */
903 [ + - ]: 146 : const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
904 : : MHD_CONNECTION_INFO_CLIENT_ADDRESS);
905 [ + - ]: 146 : struct sockaddr *so = u ? u->client_addr : 0;
906 : :
907 [ + - + - ]: 146 : if (so && so->sa_family == AF_INET) {
908 [ + - ]: 146 : sts = getnameinfo (so, sizeof (struct sockaddr_in), hostname, sizeof (hostname), servname,
909 : : sizeof (servname), NI_NUMERICHOST | NI_NUMERICSERV);
910 [ # # # # ]: 0 : } else if (so && so->sa_family == AF_INET6) {
911 [ # # ]: 0 : sts = getnameinfo (so, sizeof (struct sockaddr_in6), hostname, sizeof (hostname),
912 : : servname, sizeof (servname), NI_NUMERICHOST | NI_NUMERICSERV);
913 : : }
914 [ - + ]: 146 : if (sts != 0) {
915 : 0 : hostname[0] = servname[0] = '\0';
916 : : }
917 : :
918 : : // extract headers relevant to administration
919 [ + - + - ]: 146 : const char* user_agent = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";
920 [ + - + + ]: 146 : const char* x_forwarded_for = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";
921 : : // NB: these are untrustworthy, beware if machine-processing log files
922 : :
923 [ + - + - : 438 : return string(hostname) + string(":") + string(servname) +
+ - + - +
- ]
924 [ + - + - : 584 : string(" UA:") + header_censor(string(user_agent)) +
+ - + - +
- ]
925 [ + - + - : 438 : string(" XFF:") + header_censor(string(x_forwarded_for));
+ - + - +
- ]
926 : : }
927 : :
928 : :
929 : :
930 : : ////////////////////////////////////////////////////////////////////////
931 : :
932 : :
933 : : static void
934 : 61 : add_mhd_last_modified (struct MHD_Response *resp, time_t mtime)
935 : : {
936 : 61 : struct tm *now = gmtime (&mtime);
937 [ + - ]: 61 : if (now != NULL)
938 : : {
939 : : char datebuf[80];
940 : 61 : size_t rc = strftime (datebuf, sizeof (datebuf), "%a, %d %b %Y %T GMT", now);
941 [ + - + - ]: 61 : if (rc > 0 && rc < sizeof (datebuf))
942 [ + - ]: 61 : (void) MHD_add_response_header (resp, "Last-Modified", datebuf);
943 : : }
944 : :
945 : 61 : (void) MHD_add_response_header (resp, "Cache-Control", "public");
946 : 61 : }
947 : :
948 : :
949 : :
950 : : static struct MHD_Response*
951 : 15 : handle_buildid_f_match (bool internal_req_t,
952 : : int64_t b_mtime,
953 : : const string& b_source0,
954 : : int *result_fd)
955 : : {
956 : : (void) internal_req_t; // ignored
957 [ + - ]: 15 : int fd = open(b_source0.c_str(), O_RDONLY);
958 [ - + ]: 15 : if (fd < 0)
959 [ # # # # : 0 : throw libc_exception (errno, string("open ") + b_source0);
# # ]
960 : :
961 : : // NB: use manual close(2) in error case instead of defer_dtor, because
962 : : // in the normal case, we want to hand the fd over to libmicrohttpd for
963 : : // file transfer.
964 : :
965 : : struct stat s;
966 : 15 : int rc = fstat(fd, &s);
967 [ - + ]: 15 : if (rc < 0)
968 : : {
969 [ # # ]: 0 : close(fd);
970 [ # # # # : 0 : throw libc_exception (errno, string("fstat ") + b_source0);
# # ]
971 : : }
972 : :
973 [ - + ]: 15 : if ((int64_t) s.st_mtime != b_mtime)
974 : : {
975 [ # # ]: 0 : if (verbose)
976 [ # # # # : 0 : obatched(clog) << "mtime mismatch for " << b_source0 << endl;
# # # # ]
977 [ # # ]: 0 : close(fd);
978 : 0 : return 0;
979 : : }
980 : :
981 [ + - + - : 15 : inc_metric ("http_responses_total","result","file");
+ - + - ]
982 [ + - ]: 15 : struct MHD_Response* r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd);
983 [ - + ]: 15 : if (r == 0)
984 : : {
985 [ # # ]: 0 : if (verbose)
986 [ # # # # : 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
# # # # ]
987 [ # # ]: 0 : close(fd);
988 : : }
989 : : else
990 : : {
991 [ + - ]: 15 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
992 [ + - ]: 15 : add_mhd_last_modified (r, s.st_mtime);
993 [ + - ]: 15 : if (verbose > 1)
994 [ + - + - : 15 : obatched(clog) << "serving file " << b_source0 << endl;
+ - + - ]
995 : : /* libmicrohttpd will close it. */
996 [ + - ]: 15 : if (result_fd)
997 : 15 : *result_fd = fd;
998 : : }
999 : :
1000 : 15 : return r;
1001 : : }
1002 : :
1003 : :
1004 : : // quote all questionable characters of str for safe passage through a sh -c expansion.
1005 : : static string
1006 : 6 : shell_escape(const string& str)
1007 : : {
1008 : 6 : string y;
1009 [ + + ]: 508 : for (auto&& x : str)
1010 : : {
1011 [ + + + + ]: 502 : if (! isalnum(x) && x != '/')
1012 [ + - ]: 52 : y += "\\";
1013 [ + - ]: 502 : y += x;
1014 : : }
1015 : 6 : return y;
1016 : : }
1017 : :
1018 : :
1019 : : // PR25548: Perform POSIX / RFC3986 style path canonicalization on the input string.
1020 : : //
1021 : : // Namely:
1022 : : // // -> /
1023 : : // /foo/../ -> /
1024 : : // /./ -> /
1025 : : //
1026 : : // This mapping is done on dwarf-side source path names, which may
1027 : : // include these constructs, so we can deal with debuginfod clients
1028 : : // that accidentally canonicalize the paths.
1029 : : //
1030 : : // realpath(3) is close but not quite right, because it also resolves
1031 : : // symbolic links. Symlinks at the debuginfod server have nothing to
1032 : : // do with the build-time symlinks, thus they must not be considered.
1033 : : //
1034 : : // see also curl Curl_dedotdotify() aka RFC3986, which we mostly follow here
1035 : : // see also libc __realpath()
1036 : : // see also llvm llvm::sys::path::remove_dots()
1037 : : static string
1038 : 534 : canon_pathname (const string& input)
1039 : : {
1040 [ + - ]: 1068 : string i = input; // 5.2.4 (1)
1041 : 534 : string o;
1042 : :
1043 [ + + ]: 3392 : while (i.size() != 0)
1044 : : {
1045 : : // 5.2.4 (2) A
1046 [ + - - + ]: 2858 : if (i.substr(0,3) == "../")
1047 [ # # ]: 0 : i = i.substr(3);
1048 [ + - - + ]: 2858 : else if(i.substr(0,2) == "./")
1049 [ # # ]: 0 : i = i.substr(2);
1050 : :
1051 : : // 5.2.4 (2) B
1052 [ + - + + ]: 2858 : else if (i.substr(0,3) == "/./")
1053 [ + - ]: 55 : i = i.substr(2);
1054 [ - + ]: 2803 : else if (i == "/.")
1055 [ # # ]: 0 : i = ""; // no need to handle "/." complete-path-segment case; we're dealing with file names
1056 : :
1057 : : // 5.2.4 (2) C
1058 [ + - + + ]: 2803 : else if (i.substr(0,4) == "/../") {
1059 [ + - ]: 78 : i = i.substr(3);
1060 [ + - ]: 78 : string::size_type sl = o.rfind("/");
1061 [ + - ]: 78 : if (sl != string::npos)
1062 [ + - ]: 78 : o = o.substr(0, sl);
1063 : : else
1064 [ # # ]: 0 : o = "";
1065 [ - + ]: 2725 : } else if (i == "/..")
1066 [ # # ]: 0 : i = ""; // no need to handle "/.." complete-path-segment case; we're dealing with file names
1067 : :
1068 : : // 5.2.4 (2) D
1069 : : // no need to handle these cases; we're dealing with file names
1070 [ - + ]: 2725 : else if (i == ".")
1071 [ # # ]: 0 : i = "";
1072 [ - + ]: 2725 : else if (i == "..")
1073 [ # # ]: 0 : i = "";
1074 : :
1075 : : // POSIX special: map // to /
1076 [ + - + + ]: 2725 : else if (i.substr(0,2) == "//")
1077 [ + - ]: 14 : i = i.substr(1);
1078 : :
1079 : : // 5.2.4 (2) E
1080 : : else {
1081 [ + - + - ]: 2711 : string::size_type next_slash = i.find("/", (i[0]=='/' ? 1 : 0)); // skip first slash
1082 [ + - + - ]: 2711 : o += i.substr(0, next_slash);
1083 [ + + ]: 2711 : if (next_slash == string::npos)
1084 [ + - ]: 534 : i = "";
1085 : : else
1086 [ + - ]: 2177 : i = i.substr(next_slash);
1087 : : }
1088 : : }
1089 : :
1090 : 1068 : return o;
1091 : : }
1092 : :
1093 : :
1094 : : // Estimate available free space for a given filesystem via statfs(2).
1095 : : // Return true if the free fraction is known to be smaller than the
1096 : : // given minimum percentage. Also update a related metric.
1097 : 448 : bool statfs_free_enough_p(const string& path, const string& label, long minfree = 0)
1098 : : {
1099 : : struct statfs sfs;
1100 : 448 : int rc = statfs(path.c_str(), &sfs);
1101 [ + + ]: 449 : if (rc == 0)
1102 : : {
1103 : 448 : double s = (double) sfs.f_bavail / (double) sfs.f_blocks;
1104 [ + - + - : 448 : set_metric("filesys_free_ratio","purpose",label, s);
+ - ]
1105 : 448 : return ((s * 100.0) < minfree);
1106 : : }
1107 : 1 : return false;
1108 : : }
1109 : :
1110 : :
1111 : :
1112 : : // A map-like class that owns a cache of file descriptors (indexed by
1113 : : // file / content names).
1114 : : //
1115 : : // If only it could use fd's instead of file names ... but we can't
1116 : : // dup(2) to create independent descriptors for the same unlinked
1117 : : // files, so would have to use some goofy linux /proc/self/fd/%d
1118 : : // hack such as the following
1119 : :
1120 : : #if 0
1121 : : int superdup(int fd)
1122 : : {
1123 : : #ifdef __linux__
1124 : : char *fdpath = NULL;
1125 : : int rc = asprintf(& fdpath, "/proc/self/fd/%d", fd);
1126 : : int newfd;
1127 : : if (rc >= 0)
1128 : : newfd = open(fdpath, O_RDONLY);
1129 : : else
1130 : : newfd = -1;
1131 : : free (fdpath);
1132 : : return newfd;
1133 : : #else
1134 : : return -1;
1135 : : #endif
1136 : : }
1137 : : #endif
1138 : :
1139 : : class libarchive_fdcache
1140 : : {
1141 : : private:
1142 : : mutex fdcache_lock;
1143 : :
1144 : : struct fdcache_entry
1145 : : {
1146 : : string archive;
1147 : : string entry;
1148 : : string fd;
1149 : : double fd_size_mb; // slightly rounded up megabytes
1150 : : };
1151 : : deque<fdcache_entry> lru; // @head: most recently used
1152 : : long max_fds;
1153 : : long max_mbs;
1154 : :
1155 : : public:
1156 : 111 : void set_metrics()
1157 : : {
1158 : 111 : double total_mb = 0.0;
1159 [ + + ]: 280 : for (auto i = lru.begin(); i < lru.end(); i++)
1160 : 169 : total_mb += i->fd_size_mb;
1161 [ + - + - ]: 111 : set_metric("fdcache_bytes", (int64_t)(total_mb*1024.0*1024.0));
1162 [ + - + - ]: 111 : set_metric("fdcache_count", lru.size());
1163 : 111 : }
1164 : :
1165 : 53 : void intern(const string& a, const string& b, string fd, off_t sz, bool front_p)
1166 : : {
1167 : : {
1168 [ + - ]: 106 : unique_lock<mutex> lock(fdcache_lock);
1169 [ + + ]: 120 : for (auto i = lru.begin(); i < lru.end(); i++) // nuke preexisting copy
1170 : : {
1171 [ + + - + : 67 : if (i->archive == a && i->entry == b)
- + ]
1172 : : {
1173 : 0 : unlink (i->fd.c_str());
1174 [ # # ]: 0 : lru.erase(i);
1175 [ # # # # : 0 : inc_metric("fdcache_op_count","op","dequeue");
# # # # ]
1176 : 0 : break; // must not continue iterating
1177 : : }
1178 : : }
1179 : 53 : double mb = (sz+65535)/1048576.0; // round up to 64K block
1180 [ + - + - : 106 : fdcache_entry n = { a, b, fd, mb };
+ - ]
1181 [ + + ]: 53 : if (front_p)
1182 : : {
1183 [ + - + - : 34 : inc_metric("fdcache_op_count","op","enqueue_front");
+ - + - ]
1184 [ + - ]: 34 : lru.push_front(n);
1185 : : }
1186 : : else
1187 : : {
1188 [ + - + - : 19 : inc_metric("fdcache_op_count","op","enqueue_back");
+ - + - ]
1189 [ + - ]: 19 : lru.push_back(n);
1190 : : }
1191 [ + + ]: 53 : if (verbose > 3)
1192 [ + - + - ]: 96 : obatched(clog) << "fdcache interned a=" << a << " b=" << b
1193 [ + - + - : 48 : << " fd=" << fd << " mb=" << mb << " front=" << front_p << endl;
+ - + - +
- + - + -
+ - + - +
- ]
1194 : : }
1195 : 53 : set_metrics();
1196 : :
1197 : : // NB: we age the cache at lookup time too
1198 [ + - + - : 53 : if (statfs_free_enough_p(tmpdir, "tmpdir", fdcache_mintmp))
- + ]
1199 : : {
1200 [ # # # # : 0 : inc_metric("fdcache_op_count","op","emerg-flush");
# # # # ]
1201 [ # # # # ]: 0 : obatched(clog) << "fdcache emergency flush for filling tmpdir" << endl;
1202 : 0 : this->limit(0, 0); // emergency flush
1203 : : }
1204 [ + + ]: 53 : else if (front_p)
1205 : 34 : this->limit(max_fds, max_mbs); // age cache if required
1206 : 53 : }
1207 : :
1208 : 45 : int lookup(const string& a, const string& b)
1209 : : {
1210 : 45 : int fd = -1;
1211 : : {
1212 [ + - ]: 90 : unique_lock<mutex> lock(fdcache_lock);
1213 [ + + ]: 95 : for (auto i = lru.begin(); i < lru.end(); i++)
1214 : : {
1215 [ + + + + : 61 : if (i->archive == a && i->entry == b)
+ + ]
1216 : : { // found it; move it to head of lru
1217 [ + - ]: 22 : fdcache_entry n = *i;
1218 [ + - ]: 11 : lru.erase(i); // invalidates i, so no more iteration!
1219 [ + - ]: 11 : lru.push_front(n);
1220 [ + - + - : 11 : inc_metric("fdcache_op_count","op","requeue_front");
+ - + - ]
1221 [ + - ]: 11 : fd = open(n.fd.c_str(), O_RDONLY); // NB: no problem if dup() fails; looks like cache miss
1222 : 11 : break;
1223 : : }
1224 : : }
1225 : : }
1226 : :
1227 [ + - + - : 45 : if (statfs_free_enough_p(tmpdir, "tmpdir", fdcache_mintmp))
- + ]
1228 : : {
1229 [ # # # # : 0 : inc_metric("fdcache_op_count","op","emerg-flush");
# # # # ]
1230 [ # # ]: 0 : obatched(clog) << "fdcache emergency flush for filling tmpdir";
1231 : 0 : this->limit(0, 0); // emergency flush
1232 : : }
1233 [ + + ]: 45 : else if (fd >= 0)
1234 : 11 : this->limit(max_fds, max_mbs); // age cache if required
1235 : :
1236 : 45 : return fd;
1237 : : }
1238 : :
1239 : 53 : int probe(const string& a, const string& b) // just a cache residency check - don't modify LRU state, don't open
1240 : : {
1241 [ + - ]: 106 : unique_lock<mutex> lock(fdcache_lock);
1242 [ + + ]: 120 : for (auto i = lru.begin(); i < lru.end(); i++)
1243 : : {
1244 [ + + - + : 67 : if (i->archive == a && i->entry == b)
- + ]
1245 : : {
1246 [ # # # # : 0 : inc_metric("fdcache_op_count","op","probe_hit");
# # # # ]
1247 : 0 : return true;
1248 : : }
1249 : : }
1250 [ + - + - : 53 : inc_metric("fdcache_op_count","op","probe_miss");
+ - + - ]
1251 : 53 : return false;
1252 : : }
1253 : :
1254 : 0 : void clear(const string& a, const string& b)
1255 : : {
1256 [ # # ]: 0 : unique_lock<mutex> lock(fdcache_lock);
1257 [ # # ]: 0 : for (auto i = lru.begin(); i < lru.end(); i++)
1258 : : {
1259 [ # # # # : 0 : if (i->archive == a && i->entry == b)
# # ]
1260 : : { // found it; move it to head of lru
1261 [ # # ]: 0 : fdcache_entry n = *i;
1262 [ # # ]: 0 : lru.erase(i); // invalidates i, so no more iteration!
1263 [ # # # # : 0 : inc_metric("fdcache_op_count","op","clear");
# # # # ]
1264 : 0 : unlink (n.fd.c_str());
1265 [ # # ]: 0 : set_metrics();
1266 : 0 : return;
1267 : : }
1268 : : }
1269 : : }
1270 : :
1271 : :
1272 : 61 : void limit(long maxfds, long maxmbs, bool metrics_p = true)
1273 : : {
1274 [ + + + + : 61 : if (verbose > 3 && (this->max_fds != maxfds || this->max_mbs != maxmbs))
- + ]
1275 [ + - + - : 8 : obatched(clog) << "fdcache limited to maxfds=" << maxfds << " maxmbs=" << maxmbs << endl;
+ - + - +
- + - ]
1276 : :
1277 [ + - ]: 122 : unique_lock<mutex> lock(fdcache_lock);
1278 : 61 : this->max_fds = maxfds;
1279 : 61 : this->max_mbs = maxmbs;
1280 : :
1281 : 61 : long total_fd = 0;
1282 : 61 : double total_mb = 0.0;
1283 [ + + ]: 110 : for (auto i = lru.begin(); i < lru.end(); i++)
1284 : : {
1285 : : // accumulate totals from most recently used one going backward
1286 : 89 : total_fd ++;
1287 : 89 : total_mb += i->fd_size_mb;
1288 [ + + - + ]: 89 : if (total_fd > max_fds || total_mb > max_mbs)
1289 : : {
1290 : : // found the cut here point!
1291 : :
1292 [ + + ]: 93 : for (auto j = i; j < lru.end(); j++) // close all the fds from here on in
1293 : : {
1294 [ + + ]: 53 : if (verbose > 3)
1295 [ + - + - ]: 96 : obatched(clog) << "fdcache evicted a=" << j->archive << " b=" << j->entry
1296 [ + - + - : 48 : << " fd=" << j->fd << " mb=" << j->fd_size_mb << endl;
+ - + - +
- + - + -
+ - ]
1297 [ + + ]: 53 : if (metrics_p)
1298 [ + - + - : 47 : inc_metric("fdcache_op_count","op","evict");
+ - + - ]
1299 : 53 : unlink (j->fd.c_str());
1300 : : }
1301 : :
1302 [ + - ]: 40 : lru.erase(i, lru.end()); // erase the nodes generally
1303 : 40 : break;
1304 : : }
1305 : : }
1306 [ + + + - ]: 61 : if (metrics_p) set_metrics();
1307 : 61 : }
1308 : :
1309 : :
1310 : 3 : ~libarchive_fdcache()
1311 : 3 : {
1312 : : // unlink any fdcache entries in $TMPDIR
1313 : : // don't update metrics; those globals may be already destroyed
1314 : 3 : limit(0, 0, false);
1315 : 3 : }
1316 : : };
1317 : : static libarchive_fdcache fdcache;
1318 : :
1319 : :
1320 : : // For security/portability reasons, many distro-package archives have
1321 : : // a "./" in front of path names; others have nothing, others have
1322 : : // "/". Canonicalize them all to a single leading "/", with the
1323 : : // assumption that this matches the dwarf-derived file names too.
1324 : 137 : string canonicalized_archive_entry_pathname(struct archive_entry *e)
1325 : : {
1326 [ + - + - ]: 274 : string fn = archive_entry_pathname(e);
1327 [ - + ]: 137 : if (fn.size() == 0)
1328 : 0 : return fn;
1329 [ + - - + ]: 137 : if (fn[0] == '/')
1330 : 0 : return fn;
1331 [ + - + + ]: 137 : if (fn[0] == '.')
1332 [ + - ]: 117 : return fn.substr(1);
1333 : : else
1334 [ + - + - ]: 40 : return string("/")+fn;
1335 : : }
1336 : :
1337 : :
1338 : :
1339 : : static struct MHD_Response*
1340 : 76 : handle_buildid_r_match (bool internal_req_p,
1341 : : int64_t b_mtime,
1342 : : const string& b_source0,
1343 : : const string& b_source1,
1344 : : int *result_fd)
1345 : : {
1346 : : struct stat fs;
1347 : 76 : int rc = stat (b_source0.c_str(), &fs);
1348 [ + + ]: 76 : if (rc != 0)
1349 [ + - + - : 31 : throw libc_exception (errno, string("stat ") + b_source0);
+ - ]
1350 : :
1351 [ - + ]: 45 : if ((int64_t) fs.st_mtime != b_mtime)
1352 : : {
1353 [ # # ]: 0 : if (verbose)
1354 [ # # # # : 0 : obatched(clog) << "mtime mismatch for " << b_source0 << endl;
# # # # ]
1355 : 0 : return 0;
1356 : : }
1357 : :
1358 : : // check for a match in the fdcache first
1359 [ + - ]: 45 : int fd = fdcache.lookup(b_source0, b_source1);
1360 [ + + ]: 45 : while (fd >= 0) // got one!; NB: this is really an if() with a possible branch out to the end
1361 : : {
1362 : 11 : rc = fstat(fd, &fs);
1363 [ - + ]: 11 : if (rc < 0) // disappeared?
1364 : : {
1365 [ # # ]: 0 : if (verbose)
1366 [ # # # # : 0 : obatched(clog) << "cannot fstat fdcache " << b_source0 << endl;
# # # # ]
1367 [ # # ]: 0 : close(fd);
1368 [ # # ]: 0 : fdcache.clear(b_source0, b_source1);
1369 : 0 : break; // branch out of if "loop", to try new libarchive fetch attempt
1370 : : }
1371 : :
1372 [ + - ]: 11 : struct MHD_Response* r = MHD_create_response_from_fd (fs.st_size, fd);
1373 [ - + ]: 11 : if (r == 0)
1374 : : {
1375 [ # # ]: 0 : if (verbose)
1376 [ # # # # : 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
# # # # ]
1377 [ # # ]: 0 : close(fd);
1378 : 0 : break; // branch out of if "loop", to try new libarchive fetch attempt
1379 : : }
1380 : :
1381 [ + - + - : 11 : inc_metric ("http_responses_total","result","archive fdcache");
+ - + - ]
1382 : :
1383 [ + - ]: 11 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
1384 [ + - ]: 11 : add_mhd_last_modified (r, fs.st_mtime);
1385 [ + - ]: 11 : if (verbose > 1)
1386 [ + - + - : 11 : obatched(clog) << "serving fdcache archive " << b_source0 << " file " << b_source1 << endl;
+ - + - +
- + - ]
1387 : : /* libmicrohttpd will close it. */
1388 [ + - ]: 11 : if (result_fd)
1389 : 11 : *result_fd = fd;
1390 : 11 : return r;
1391 : : // NB: see, we never go around the 'loop' more than once
1392 : : }
1393 : :
1394 : : // no match ... grumble, must process the archive
1395 [ + - ]: 68 : string archive_decoder = "/dev/null";
1396 [ + - ]: 68 : string archive_extension = "";
1397 [ + + ]: 136 : for (auto&& arch : scan_archives)
1398 [ + - + + ]: 102 : if (string_endswith(b_source0, arch.first))
1399 : : {
1400 [ + - ]: 34 : archive_extension = arch.first;
1401 [ + - ]: 34 : archive_decoder = arch.second;
1402 : : }
1403 : : FILE* fp;
1404 : : defer_dtor<FILE*,int>::dtor_fn dfn;
1405 [ + + ]: 34 : if (archive_decoder != "cat")
1406 : : {
1407 [ + - + - : 9 : string popen_cmd = archive_decoder + " " + shell_escape(b_source0);
+ - ]
1408 [ + - ]: 3 : fp = popen (popen_cmd.c_str(), "r"); // "e" O_CLOEXEC?
1409 : 3 : dfn = pclose;
1410 [ - + ]: 3 : if (fp == NULL)
1411 [ # # # # : 0 : throw libc_exception (errno, string("popen ") + popen_cmd);
# # ]
1412 : : }
1413 : : else
1414 : : {
1415 [ + - ]: 31 : fp = fopen (b_source0.c_str(), "r");
1416 : 31 : dfn = fclose;
1417 [ - + ]: 31 : if (fp == NULL)
1418 [ # # # # : 0 : throw libc_exception (errno, string("fopen ") + b_source0);
# # ]
1419 : : }
1420 : 68 : defer_dtor<FILE*,int> fp_closer (fp, dfn);
1421 : :
1422 : : struct archive *a;
1423 [ + - ]: 34 : a = archive_read_new();
1424 [ - + ]: 34 : if (a == NULL)
1425 [ # # # # ]: 0 : throw archive_exception("cannot create archive reader");
1426 : 34 : defer_dtor<struct archive*,int> archive_closer (a, archive_read_free);
1427 : :
1428 [ + - ]: 34 : rc = archive_read_support_format_all(a);
1429 [ - + ]: 34 : if (rc != ARCHIVE_OK)
1430 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all format");
1431 [ + - ]: 34 : rc = archive_read_support_filter_all(a);
1432 [ - + ]: 34 : if (rc != ARCHIVE_OK)
1433 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all filters");
1434 : :
1435 [ + - ]: 34 : rc = archive_read_open_FILE (a, fp);
1436 [ - + ]: 34 : if (rc != ARCHIVE_OK)
1437 [ # # # # ]: 0 : throw archive_exception(a, "cannot open archive from pipe");
1438 : :
1439 : : // archive traversal is in three stages, no, four stages:
1440 : : // 1) skip entries whose names do not match the requested one
1441 : : // 2) extract the matching entry name (set r = result)
1442 : : // 3) extract some number of prefetched entries (just into fdcache)
1443 : : // 4) abort any further processing
1444 : 34 : struct MHD_Response* r = 0; // will set in stage 2
1445 [ + + ]: 34 : unsigned prefetch_count =
1446 : : internal_req_p ? 0 : fdcache_prefetch; // will decrement in stage 3
1447 : :
1448 [ + + + + ]: 340 : while(r == 0 || prefetch_count > 0) // stage 1, 2, or 3
1449 : : {
1450 [ - + ]: 336 : if (interrupted)
1451 : 30 : break;
1452 : :
1453 : : struct archive_entry *e;
1454 [ + - ]: 336 : rc = archive_read_next_header (a, &e);
1455 [ + + ]: 336 : if (rc != ARCHIVE_OK)
1456 : 30 : break;
1457 : :
1458 [ + - + + ]: 306 : if (! S_ISREG(archive_entry_mode (e))) // skip non-files completely
1459 : 306 : continue;
1460 : :
1461 [ + - ]: 68 : string fn = canonicalized_archive_entry_pathname (e);
1462 [ + + + + : 68 : if ((r == 0) && (fn != b_source1)) // stage 1
+ + ]
1463 : 15 : continue;
1464 : :
1465 [ + - - + ]: 53 : if (fdcache.probe (b_source0, fn)) // skip if already interned
1466 : 0 : continue;
1467 : :
1468 : : // extract this file to a temporary file
1469 : 53 : char* tmppath = NULL;
1470 : 53 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
1471 [ - + ]: 53 : if (rc < 0)
1472 [ # # # # ]: 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
1473 : 53 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
1474 [ + - ]: 53 : fd = mkstemp (tmppath);
1475 [ - + ]: 53 : if (fd < 0)
1476 [ # # # # ]: 0 : throw libc_exception (errno, "cannot create temporary file");
1477 : : // NB: don't unlink (tmppath), as fdcache will take charge of it.
1478 : :
1479 : : // NB: this can take many uninterruptible seconds for a huge file
1480 [ + - ]: 53 : rc = archive_read_data_into_fd (a, fd);
1481 [ - + ]: 53 : if (rc != ARCHIVE_OK) // e.g. ENOSPC!
1482 : : {
1483 [ # # ]: 0 : close (fd);
1484 : 0 : unlink (tmppath);
1485 [ # # # # ]: 0 : throw archive_exception(a, "cannot extract file");
1486 : : }
1487 : :
1488 : : // Set the mtime so the fdcache file mtimes, even prefetched ones,
1489 : : // propagate to future webapi clients.
1490 : : struct timeval tvs[2];
1491 [ + - ]: 53 : tvs[0].tv_sec = tvs[1].tv_sec = archive_entry_mtime(e);
1492 : 53 : tvs[0].tv_usec = tvs[1].tv_usec = 0;
1493 : 53 : (void) futimes (fd, tvs); /* best effort */
1494 : :
1495 [ + + ]: 53 : if (r != 0) // stage 3
1496 : : {
1497 : : // NB: now we know we have a complete reusable file; make fdcache
1498 : : // responsible for unlinking it later.
1499 [ + - + - ]: 19 : fdcache.intern(b_source0, fn,
1500 : : tmppath, archive_entry_size(e),
1501 [ + - ]: 38 : false); // prefetched ones go to back of lru
1502 : 19 : prefetch_count --;
1503 [ + - ]: 19 : close (fd); // we're not saving this fd to make a mhd-response from!
1504 : 19 : continue;
1505 : : }
1506 : :
1507 : : // NB: now we know we have a complete reusable file; make fdcache
1508 : : // responsible for unlinking it later.
1509 [ + - + - ]: 34 : fdcache.intern(b_source0, b_source1,
1510 : : tmppath, archive_entry_size(e),
1511 [ + - ]: 68 : true); // requested ones go to the front of lru
1512 : :
1513 [ + - + - : 34 : inc_metric ("http_responses_total","result",archive_extension + " archive");
+ - + - ]
1514 [ + - + - ]: 34 : r = MHD_create_response_from_fd (archive_entry_size(e), fd);
1515 [ - + ]: 34 : if (r == 0)
1516 : : {
1517 [ # # ]: 0 : if (verbose)
1518 [ # # # # : 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
# # # # ]
1519 [ # # ]: 0 : close(fd);
1520 : 0 : break; // assume no chance of better luck around another iteration; no other copies of same file
1521 : : }
1522 : : else
1523 : : {
1524 [ + - ]: 34 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
1525 [ + - + - ]: 34 : add_mhd_last_modified (r, archive_entry_mtime(e));
1526 [ + - ]: 34 : if (verbose > 1)
1527 [ + - + - : 34 : obatched(clog) << "serving archive " << b_source0 << " file " << b_source1 << endl;
+ - + - +
- + - ]
1528 : : /* libmicrohttpd will close it. */
1529 [ + - ]: 34 : if (result_fd)
1530 : 34 : *result_fd = fd;
1531 : 34 : continue;
1532 : : }
1533 : : }
1534 : :
1535 : : // XXX: rpm/file not found: delete this R entry?
1536 : 34 : return r;
1537 : : }
1538 : :
1539 : :
1540 : : static struct MHD_Response*
1541 : 91 : handle_buildid_match (bool internal_req_p,
1542 : : int64_t b_mtime,
1543 : : const string& b_stype,
1544 : : const string& b_source0,
1545 : : const string& b_source1,
1546 : : int *result_fd)
1547 : : {
1548 : : try
1549 : : {
1550 [ + + ]: 91 : if (b_stype == "F")
1551 [ + - ]: 15 : return handle_buildid_f_match(internal_req_p, b_mtime, b_source0, result_fd);
1552 [ + - ]: 76 : else if (b_stype == "R")
1553 [ + + ]: 76 : return handle_buildid_r_match(internal_req_p, b_mtime, b_source0, b_source1, result_fd);
1554 : : }
1555 [ - + ]: 62 : catch (const reportable_exception &e)
1556 : : {
1557 [ + - ]: 31 : e.report(clog);
1558 : : // Report but swallow libc etc. errors here; let the caller
1559 : : // iterate to other matches of the content.
1560 : : }
1561 : :
1562 : 31 : return 0;
1563 : : }
1564 : :
1565 : :
1566 : : static int
1567 : 6 : debuginfod_find_progress (debuginfod_client *, long a, long b)
1568 : : {
1569 [ - + ]: 6 : if (verbose > 4)
1570 [ # # # # : 0 : obatched(clog) << "federated debuginfod progress=" << a << "/" << b << endl;
# # # # #
# ]
1571 : :
1572 : 6 : return interrupted;
1573 : : }
1574 : :
1575 : :
1576 : : // a little lru pool of debuginfod_client*s for reuse between query threads
1577 : :
1578 : : mutex dc_pool_lock;
1579 : : deque<debuginfod_client*> dc_pool;
1580 : :
1581 : 9 : debuginfod_client* debuginfod_pool_begin()
1582 : : {
1583 [ + - ]: 18 : unique_lock<mutex> lock(dc_pool_lock);
1584 [ + + ]: 9 : if (dc_pool.size() > 0)
1585 : : {
1586 [ + - + - : 5 : inc_metric("dc_pool_op_count","op","begin-reuse");
+ - + - ]
1587 : 5 : debuginfod_client *c = dc_pool.front();
1588 : 5 : dc_pool.pop_front();
1589 : 5 : return c;
1590 : : }
1591 [ + - + - : 4 : inc_metric("dc_pool_op_count","op","begin-new");
+ - + - ]
1592 [ + - ]: 4 : return debuginfod_begin();
1593 : : }
1594 : :
1595 : :
1596 : 5 : void debuginfod_pool_groom()
1597 : : {
1598 [ + - ]: 10 : unique_lock<mutex> lock(dc_pool_lock);
1599 [ + + ]: 7 : while (dc_pool.size() > 0)
1600 : : {
1601 [ + - + - : 2 : inc_metric("dc_pool_op_count","op","end");
+ - + - ]
1602 [ + - ]: 2 : debuginfod_end(dc_pool.front());
1603 : 2 : dc_pool.pop_front();
1604 : : }
1605 : 5 : }
1606 : :
1607 : :
1608 : 9 : void debuginfod_pool_end(debuginfod_client* c)
1609 : : {
1610 [ + - ]: 18 : unique_lock<mutex> lock(dc_pool_lock);
1611 [ + - + - : 9 : inc_metric("dc_pool_op_count","op","end-save");
+ - + - ]
1612 [ + - ]: 9 : dc_pool.push_front(c); // accelerate reuse, vs. push_back
1613 : 9 : }
1614 : :
1615 : :
1616 : : static struct MHD_Response*
1617 : 71 : handle_buildid (MHD_Connection* conn,
1618 : : const string& buildid /* unsafe */,
1619 : : const string& artifacttype /* unsafe */,
1620 : : const string& suffix /* unsafe */,
1621 : : int *result_fd)
1622 : : {
1623 : : // validate artifacttype
1624 : 142 : string atype_code;
1625 [ + + + - ]: 71 : if (artifacttype == "debuginfo") atype_code = "D";
1626 [ + + + - ]: 32 : else if (artifacttype == "executable") atype_code = "E";
1627 [ + + + - ]: 15 : else if (artifacttype == "source") atype_code = "S";
1628 [ + - + - ]: 1 : else throw reportable_exception("invalid artifacttype");
1629 : :
1630 [ + - + - : 70 : inc_metric("http_requests_total", "type", artifacttype);
+ - ]
1631 : :
1632 [ + + - + : 70 : if (atype_code == "S" && suffix == "")
- + ]
1633 [ # # # # ]: 0 : throw reportable_exception("invalid source suffix");
1634 : :
1635 : : // validate buildid
1636 : 70 : if ((buildid.size() < 2) || // not empty
1637 [ + - + - : 140 : (buildid.size() % 2) || // even number
- + - + ]
1638 : 70 : (buildid.find_first_not_of("0123456789abcdef") != string::npos)) // pure tasty lowercase hex
1639 [ # # # # ]: 0 : throw reportable_exception("invalid buildid");
1640 : :
1641 [ + - ]: 70 : if (verbose > 1)
1642 [ + - + - ]: 140 : obatched(clog) << "searching for buildid=" << buildid << " artifacttype=" << artifacttype
1643 [ + - + - : 70 : << " suffix=" << suffix << endl;
+ - + - +
- + - ]
1644 : :
1645 : : // If invoked from the scanner threads, use the scanners' read-write
1646 : : // connection. Otherwise use the web query threads' read-only connection.
1647 [ + + ]: 70 : sqlite3 *thisdb = (conn == 0) ? db : dbq;
1648 : :
1649 : 70 : sqlite_ps *pp = 0;
1650 : :
1651 [ + + ]: 70 : if (atype_code == "D")
1652 : : {
1653 : 39 : pp = new sqlite_ps (thisdb, "mhd-query-d",
1654 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_d where buildid = ? "
1655 [ + - + - : 39 : "order by mtime desc");
+ - + - ]
1656 [ + - ]: 39 : pp->reset();
1657 [ + - ]: 39 : pp->bind(1, buildid);
1658 : : }
1659 [ + + ]: 31 : else if (atype_code == "E")
1660 : : {
1661 : 17 : pp = new sqlite_ps (thisdb, "mhd-query-e",
1662 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_e where buildid = ? "
1663 [ + - + - : 17 : "order by mtime desc");
+ - + - ]
1664 [ + - ]: 17 : pp->reset();
1665 [ + - ]: 17 : pp->bind(1, buildid);
1666 : : }
1667 [ + - ]: 14 : else if (atype_code == "S")
1668 : : {
1669 : : // PR25548
1670 : : // Incoming source queries may come in with either dwarf-level OR canonicalized paths.
1671 : : // We let the query pass with either one.
1672 : :
1673 : 14 : pp = new sqlite_ps (thisdb, "mhd-query-s",
1674 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_s where buildid = ? and artifactsrc in (?,?) "
1675 [ + - + - : 14 : "order by sharedprefix(source0,source0ref) desc, mtime desc");
+ - + - ]
1676 [ + - ]: 14 : pp->reset();
1677 [ + - ]: 14 : pp->bind(1, buildid);
1678 : : // NB: we don't store the non-canonicalized path names any more, but old databases
1679 : : // might have them (and no canon ones), so we keep searching for both.
1680 [ + - ]: 14 : pp->bind(2, suffix);
1681 [ + - + - ]: 14 : pp->bind(3, canon_pathname(suffix));
1682 : : }
1683 : 140 : unique_ptr<sqlite_ps> ps_closer(pp); // release pp if exception or return
1684 : :
1685 : : // consume all the rows
1686 : : while (1)
1687 : : {
1688 [ + - ]: 101 : int rc = pp->step();
1689 [ + + ]: 101 : if (rc == SQLITE_DONE) break;
1690 [ + + ]: 92 : if (rc != SQLITE_ROW)
1691 [ + - + - ]: 1 : throw sqlite_exception(rc, "step");
1692 : :
1693 [ + - ]: 91 : int64_t b_mtime = sqlite3_column_int64 (*pp, 0);
1694 [ + - + - : 91 : string b_stype = string((const char*) sqlite3_column_text (*pp, 1) ?: ""); /* by DDL may not be NULL */
+ - ]
1695 [ + - + - : 91 : string b_source0 = string((const char*) sqlite3_column_text (*pp, 2) ?: ""); /* may be NULL */
+ - ]
1696 [ + - + + : 91 : string b_source1 = string((const char*) sqlite3_column_text (*pp, 3) ?: ""); /* may be NULL */
+ - ]
1697 : :
1698 [ + - ]: 91 : if (verbose > 1)
1699 [ + - + - : 182 : obatched(clog) << "found mtime=" << b_mtime << " stype=" << b_stype
+ - ]
1700 [ + - + - : 91 : << " source0=" << b_source0 << " source1=" << b_source1 << endl;
+ - + - +
- + - +
- ]
1701 : :
1702 : : // Try accessing the located match.
1703 : : // XXX: in case of multiple matches, attempt them in parallel?
1704 [ + - ]: 91 : auto r = handle_buildid_match (conn ? false : true,
1705 : : b_mtime, b_stype, b_source0, b_source1, result_fd);
1706 [ + + ]: 91 : if (r)
1707 : 60 : return r;
1708 : 31 : }
1709 [ + - ]: 9 : pp->reset();
1710 : :
1711 : : // We couldn't find it in the database. Last ditch effort
1712 : : // is to defer to other debuginfo servers.
1713 : :
1714 : 9 : int fd = -1;
1715 [ + - ]: 9 : debuginfod_client *client = debuginfod_pool_begin ();
1716 [ + - ]: 9 : if (client != NULL)
1717 : : {
1718 [ + - ]: 9 : debuginfod_set_progressfn (client, & debuginfod_find_progress);
1719 : :
1720 [ + - ]: 9 : if (conn)
1721 : : {
1722 : : // Transcribe incoming User-Agent:
1723 [ + - + - : 18 : string ua = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";
+ - ]
1724 [ + - + - ]: 27 : string ua_complete = string("User-Agent: ") + ua;
1725 [ + - ]: 9 : debuginfod_add_http_header (client, ua_complete.c_str());
1726 : :
1727 : : // Compute larger XFF:, for avoiding info loss during
1728 : : // federation, and for future cyclicity detection.
1729 [ + - + + : 18 : string xff = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";
+ - ]
1730 [ + + ]: 9 : if (xff != "")
1731 [ + - + - ]: 2 : xff += string(", "); // comma separated list
1732 : :
1733 : : // Compute the client's numeric IP address only - so can't merge with conninfo()
1734 [ + - ]: 9 : const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
1735 : : MHD_CONNECTION_INFO_CLIENT_ADDRESS);
1736 [ + - ]: 9 : struct sockaddr *so = u ? u->client_addr : 0;
1737 : 9 : char hostname[256] = ""; // RFC1035
1738 [ + - + - ]: 9 : if (so && so->sa_family == AF_INET)
1739 [ + - ]: 9 : (void) getnameinfo (so, sizeof (struct sockaddr_in), hostname, sizeof (hostname), NULL, 0,
1740 : : NI_NUMERICHOST);
1741 [ # # # # ]: 0 : else if (so && so->sa_family == AF_INET6)
1742 [ # # ]: 0 : (void) getnameinfo (so, sizeof (struct sockaddr_in6), hostname, sizeof (hostname), NULL, 0,
1743 : : NI_NUMERICHOST);
1744 : :
1745 [ + - + - : 36 : string xff_complete = string("X-Forwarded-For: ")+xff+string(hostname);
+ - + - ]
1746 [ + - ]: 9 : debuginfod_add_http_header (client, xff_complete.c_str());
1747 : : }
1748 : :
1749 [ + + ]: 9 : if (artifacttype == "debuginfo")
1750 [ + - ]: 8 : fd = debuginfod_find_debuginfo (client,
1751 : 8 : (const unsigned char*) buildid.c_str(),
1752 : : 0, NULL);
1753 [ + - ]: 1 : else if (artifacttype == "executable")
1754 [ + - ]: 1 : fd = debuginfod_find_executable (client,
1755 : 1 : (const unsigned char*) buildid.c_str(),
1756 : : 0, NULL);
1757 [ # # ]: 0 : else if (artifacttype == "source")
1758 [ # # ]: 0 : fd = debuginfod_find_source (client,
1759 : 0 : (const unsigned char*) buildid.c_str(),
1760 : : 0, suffix.c_str(), NULL);
1761 : : }
1762 : : else
1763 : 0 : fd = -errno; /* Set by debuginfod_begin. */
1764 [ + - ]: 9 : debuginfod_pool_end (client);
1765 : :
1766 [ + + ]: 9 : if (fd >= 0)
1767 : : {
1768 [ + - + - : 1 : inc_metric ("http_responses_total","result","upstream");
+ - + - ]
1769 : : struct stat s;
1770 : 1 : int rc = fstat (fd, &s);
1771 [ + - ]: 1 : if (rc == 0)
1772 : : {
1773 [ + - ]: 1 : auto r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd);
1774 [ + - ]: 1 : if (r)
1775 : : {
1776 [ + - ]: 1 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
1777 [ + - ]: 1 : add_mhd_last_modified (r, s.st_mtime);
1778 [ + - ]: 1 : if (verbose > 1)
1779 [ + - + - : 1 : obatched(clog) << "serving file from upstream debuginfod/cache" << endl;
+ - ]
1780 [ + - ]: 1 : if (result_fd)
1781 : 1 : *result_fd = fd;
1782 : 1 : return r; // NB: don't close fd; libmicrohttpd will
1783 : : }
1784 : : }
1785 [ # # ]: 0 : close (fd);
1786 : : }
1787 : : else
1788 [ + + - ]: 8 : switch(fd)
1789 : : {
1790 : 6 : case -ENOSYS:
1791 : 6 : break;
1792 : 2 : case -ENOENT:
1793 : 2 : break;
1794 : 0 : default: // some more tricky error
1795 [ # # # # ]: 0 : throw libc_exception(-fd, "upstream debuginfod query failed");
1796 : : }
1797 : :
1798 [ + - + - ]: 8 : throw reportable_exception(MHD_HTTP_NOT_FOUND, "not found");
1799 : : }
1800 : :
1801 : :
1802 : : ////////////////////////////////////////////////////////////////////////
1803 : :
1804 : : static map<string,double> metrics; // arbitrary data for /metrics query
1805 : : // NB: store int64_t since all our metrics are integers; prometheus accepts double
1806 : : static mutex metrics_lock;
1807 : : // NB: these objects get released during the process exit via global dtors
1808 : : // do not call them from within other global dtors
1809 : :
1810 : : // utility function for assembling prometheus-compatible
1811 : : // name="escaped-value" strings
1812 : : // https://prometheus.io/docs/instrumenting/exposition_formats/
1813 : : static string
1814 : 14329 : metric_label(const string& name, const string& value)
1815 : : {
1816 : 14329 : string x = name + "=\"";
1817 [ + + ]: 204867 : for (auto&& c : value)
1818 [ - - - + ]: 190593 : switch(c)
1819 : : {
1820 [ # # ]: 0 : case '\\': x += "\\\\"; break;
1821 [ # # ]: 0 : case '\"': x += "\\\""; break;
1822 [ # # ]: 0 : case '\n': x += "\\n"; break;
1823 [ + - ]: 190593 : default: x += c; break;
1824 : : }
1825 [ + - ]: 14325 : x += "\"";
1826 : 14330 : return x;
1827 : : }
1828 : :
1829 : :
1830 : : // add prometheus-format metric name + label tuple (if any) + value
1831 : :
1832 : : static void
1833 : 228 : set_metric(const string& metric, double value)
1834 : : {
1835 [ + - ]: 228 : unique_lock<mutex> lock(metrics_lock);
1836 [ + - ]: 228 : metrics[metric] = value;
1837 : 228 : }
1838 : : #if 0 /* unused */
1839 : : static void
1840 : : inc_metric(const string& metric)
1841 : : {
1842 : : unique_lock<mutex> lock(metrics_lock);
1843 : : metrics[metric] ++;
1844 : : }
1845 : : #endif
1846 : : static void
1847 : 885 : set_metric(const string& metric,
1848 : : const string& lname, const string& lvalue,
1849 : : double value)
1850 : : {
1851 [ + - + - : 2657 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
+ - + - ]
1852 [ + - ]: 886 : unique_lock<mutex> lock(metrics_lock);
1853 [ + - ]: 886 : metrics[key] = value;
1854 : 886 : }
1855 : :
1856 : : static void
1857 : 7127 : inc_metric(const string& metric,
1858 : : const string& lname, const string& lvalue)
1859 : : {
1860 [ + - + - : 21381 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
+ - + - ]
1861 [ + - ]: 7127 : unique_lock<mutex> lock(metrics_lock);
1862 [ + - ]: 7127 : metrics[key] ++;
1863 : 7127 : }
1864 : : static void
1865 : 6319 : add_metric(const string& metric,
1866 : : const string& lname, const string& lvalue,
1867 : : double value)
1868 : : {
1869 [ + - + - : 18955 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
+ - + - ]
1870 [ + - ]: 6319 : unique_lock<mutex> lock(metrics_lock);
1871 [ + - ]: 6319 : metrics[key] += value;
1872 : 6319 : }
1873 : : #if 0
1874 : : static void
1875 : : add_metric(const string& metric,
1876 : : double value)
1877 : : {
1878 : : unique_lock<mutex> lock(metrics_lock);
1879 : : metrics[metric] += value;
1880 : : }
1881 : : #endif
1882 : :
1883 : :
1884 : : // and more for higher arity labels if needed
1885 : :
1886 : :
1887 : : static struct MHD_Response*
1888 : 81 : handle_metrics (off_t* size)
1889 : : {
1890 [ + - ]: 162 : stringstream o;
1891 : : {
1892 [ + - ]: 162 : unique_lock<mutex> lock(metrics_lock);
1893 [ + + ]: 11186 : for (auto&& i : metrics)
1894 : 11105 : o << i.first
1895 : : << " "
1896 [ + - + - : 11105 : << std::setprecision(std::numeric_limits<double>::digits10 + 1)
+ - ]
1897 [ + - ]: 11105 : << i.second
1898 [ + - ]: 11105 : << endl;
1899 : : }
1900 [ + - ]: 81 : const string& os = o.str();
1901 [ + - ]: 162 : MHD_Response* r = MHD_create_response_from_buffer (os.size(),
1902 : 81 : (void*) os.c_str(),
1903 : : MHD_RESPMEM_MUST_COPY);
1904 : 81 : *size = os.size();
1905 [ + - ]: 81 : MHD_add_response_header (r, "Content-Type", "text/plain");
1906 : 162 : return r;
1907 : : }
1908 : :
1909 : : static struct MHD_Response*
1910 : 0 : handle_root (off_t* size)
1911 : : {
1912 [ # # # # : 0 : static string version = "debuginfod (" + string (PACKAGE_NAME) + ") "
# # ]
1913 [ # # # # : 0 : + string (PACKAGE_VERSION);
# # # # #
# ]
1914 : 0 : MHD_Response* r = MHD_create_response_from_buffer (version.size (),
1915 : 0 : (void *) version.c_str (),
1916 : : MHD_RESPMEM_PERSISTENT);
1917 : 0 : *size = version.size ();
1918 : 0 : MHD_add_response_header (r, "Content-Type", "text/plain");
1919 : 0 : return r;
1920 : : }
1921 : :
1922 : :
1923 : : ////////////////////////////////////////////////////////////////////////
1924 : :
1925 : :
1926 : : /* libmicrohttpd callback */
1927 : : static MHD_RESULT
1928 : 292 : handler_cb (void * /*cls*/,
1929 : : struct MHD_Connection *connection,
1930 : : const char *url,
1931 : : const char *method,
1932 : : const char * /*version*/,
1933 : : const char * /*upload_data*/,
1934 : : size_t * /*upload_data_size*/,
1935 : : void ** ptr)
1936 : : {
1937 : 292 : struct MHD_Response *r = NULL;
1938 [ + - ]: 584 : string url_copy = url;
1939 : :
1940 : : /* libmicrohttpd always makes (at least) two callbacks: once just
1941 : : past the headers, and one after the request body is finished
1942 : : being received. If we process things early (first callback) and
1943 : : queue a response, libmicrohttpd would suppress http keep-alive
1944 : : (via connection->read_closed = true). */
1945 : : static int aptr; /* just some random object to use as a flag */
1946 [ + + ]: 292 : if (&aptr != *ptr)
1947 : : {
1948 : : /* do never respond on first call */
1949 : 146 : *ptr = &aptr;
1950 : 146 : return MHD_YES;
1951 : : }
1952 : 146 : *ptr = NULL; /* reset when done */
1953 : :
1954 : : #if MHD_VERSION >= 0x00097002
1955 : : enum MHD_Result rc;
1956 : : #else
1957 : : int rc = MHD_NO; // mhd
1958 : : #endif
1959 : 146 : int http_code = 500;
1960 : 146 : off_t http_size = -1;
1961 : : struct timespec ts_start, ts_end;
1962 : 146 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
1963 : :
1964 : : try
1965 : : {
1966 [ + - - + ]: 146 : if (string(method) != "GET")
1967 [ # # # # ]: 0 : throw reportable_exception(400, "we support GET only");
1968 : :
1969 : : /* Start decoding the URL. */
1970 : 146 : size_t slash1 = url_copy.find('/', 1);
1971 [ + - ]: 292 : string url1 = url_copy.substr(0, slash1); // ok even if slash1 not found
1972 : :
1973 [ + + + - : 146 : if (slash1 != string::npos && url1 == "/buildid")
+ + ]
1974 : : {
1975 [ + - + - : 315 : tmp_inc_metric m ("thread_busy", "role", "http-buildid");
+ - + - ]
1976 : 63 : size_t slash2 = url_copy.find('/', slash1+1);
1977 [ - + ]: 63 : if (slash2 == string::npos)
1978 [ # # # # ]: 0 : throw reportable_exception("/buildid/ webapi error, need buildid");
1979 : :
1980 [ + - ]: 126 : string buildid = url_copy.substr(slash1+1, slash2-slash1-1);
1981 : :
1982 : 63 : size_t slash3 = url_copy.find('/', slash2+1);
1983 : 136 : string artifacttype, suffix;
1984 [ + + ]: 63 : if (slash3 == string::npos)
1985 : : {
1986 [ + - ]: 49 : artifacttype = url_copy.substr(slash2+1);
1987 [ + - ]: 49 : suffix = "";
1988 : : }
1989 : : else
1990 : : {
1991 [ + - ]: 14 : artifacttype = url_copy.substr(slash2+1, slash3-slash2-1);
1992 [ + - ]: 14 : suffix = url_copy.substr(slash3); // include the slash in the suffix
1993 : : }
1994 : :
1995 : : // get the resulting fd so we can report its size
1996 : : int fd;
1997 [ + + ]: 63 : r = handle_buildid(connection, buildid, artifacttype, suffix, &fd);
1998 [ + - ]: 53 : if (r)
1999 : : {
2000 : : struct stat fs;
2001 [ + - ]: 53 : if (fstat(fd, &fs) == 0)
2002 : 53 : http_size = fs.st_size;
2003 : : // libmicrohttpd will close (fd);
2004 : : }
2005 : : }
2006 [ + + ]: 83 : else if (url1 == "/metrics")
2007 : : {
2008 [ + - + - : 324 : tmp_inc_metric m ("thread_busy", "role", "http-metrics");
+ - + - ]
2009 [ + - + - : 81 : inc_metric("http_requests_total", "type", "metrics");
+ - + - ]
2010 [ + - ]: 81 : r = handle_metrics(& http_size);
2011 : : }
2012 [ - + ]: 2 : else if (url1 == "/")
2013 : : {
2014 [ # # # # : 0 : inc_metric("http_requests_total", "type", "/");
# # # # ]
2015 [ # # ]: 0 : r = handle_root(& http_size);
2016 : : }
2017 : : else
2018 [ + - + - : 2 : throw reportable_exception("webapi error, unrecognized '" + url1 + "'");
+ - ]
2019 : :
2020 [ - + ]: 134 : if (r == 0)
2021 [ # # # # ]: 0 : throw reportable_exception("internal error, missing response");
2022 : :
2023 [ + - ]: 134 : rc = MHD_queue_response (connection, MHD_HTTP_OK, r);
2024 : 134 : http_code = MHD_HTTP_OK;
2025 [ + - ]: 134 : MHD_destroy_response (r);
2026 : : }
2027 [ - + ]: 12 : catch (const reportable_exception& e)
2028 : : {
2029 [ + - + - : 12 : inc_metric("http_responses_total","result","error");
+ - + - ]
2030 [ + - ]: 12 : e.report(clog);
2031 : 12 : http_code = e.code;
2032 : 12 : http_size = e.message.size();
2033 [ + - ]: 12 : rc = e.mhd_send_response (connection);
2034 : : }
2035 : :
2036 : 146 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
2037 : 146 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
2038 [ + - + - : 292 : obatched(clog) << conninfo(connection)
+ - ]
2039 : : << ' ' << method << ' ' << url
2040 [ + - + - : 146 : << ' ' << http_code << ' ' << http_size
+ - + - +
- + - +
- ]
2041 [ + - + - : 146 : << ' ' << (int)(deltas*1000) << "ms"
+ - + - ]
2042 [ + - ]: 146 : << endl;
2043 : :
2044 : : // related prometheus metrics
2045 [ + - ]: 146 : string http_code_str = to_string(http_code);
2046 [ + - ]: 146 : if (http_size >= 0)
2047 [ + - ]: 146 : add_metric("http_responses_transfer_bytes_sum","code",http_code_str,
2048 [ + - + - ]: 292 : http_size);
2049 [ + - + - : 146 : inc_metric("http_responses_transfer_bytes_count","code",http_code_str);
+ - ]
2050 : :
2051 [ + - ]: 146 : add_metric("http_responses_duration_milliseconds_sum","code",http_code_str,
2052 [ + - + - ]: 292 : deltas*1000); // prometheus prefers _seconds and floating point
2053 [ + - + - : 146 : inc_metric("http_responses_duration_milliseconds_count","code",http_code_str);
+ - ]
2054 : :
2055 : 146 : return rc;
2056 : : }
2057 : :
2058 : :
2059 : : ////////////////////////////////////////////////////////////////////////
2060 : : // borrowed originally from src/nm.c get_local_names()
2061 : :
2062 : : static void
2063 : 23 : dwarf_extract_source_paths (Elf *elf, set<string>& debug_sourcefiles)
2064 : : noexcept // no exceptions - so we can simplify the altdbg resource release at end
2065 : : {
2066 : 23 : Dwarf* dbg = dwarf_begin_elf (elf, DWARF_C_READ, NULL);
2067 [ - + ]: 23 : if (dbg == NULL)
2068 : 0 : return;
2069 : :
2070 : 23 : Dwarf* altdbg = NULL;
2071 : 23 : int altdbg_fd = -1;
2072 : :
2073 : : // DWZ handling: if we have an unsatisfied debug-alt-link, add an
2074 : : // empty string into the outgoing sourcefiles set, so the caller
2075 : : // should know that our data is incomplete.
2076 : : const char *alt_name_p;
2077 : : const void *alt_build_id; // elfutils-owned memory
2078 : 23 : ssize_t sz = dwelf_dwarf_gnu_debugaltlink (dbg, &alt_name_p, &alt_build_id);
2079 [ + + ]: 23 : if (sz > 0) // got one!
2080 : : {
2081 : 16 : string buildid;
2082 : 8 : unsigned char* build_id_bytes = (unsigned char*) alt_build_id;
2083 [ + + ]: 168 : for (ssize_t idx=0; idx<sz; idx++)
2084 : : {
2085 : 160 : buildid += "0123456789abcdef"[build_id_bytes[idx] >> 4];
2086 : 160 : buildid += "0123456789abcdef"[build_id_bytes[idx] & 0xf];
2087 : : }
2088 : :
2089 [ + - ]: 8 : if (verbose > 3)
2090 : 8 : obatched(clog) << "Need altdebug buildid=" << buildid << endl;
2091 : :
2092 : : // but is it unsatisfied the normal elfutils ways?
2093 : 8 : Dwarf* alt = dwarf_getalt (dbg);
2094 [ + - ]: 8 : if (alt == NULL)
2095 : : {
2096 : : // Yup, unsatisfied the normal way. Maybe we can satisfy it
2097 : : // from our own debuginfod database.
2098 : : int alt_fd;
2099 : 8 : struct MHD_Response *r = 0;
2100 : : try
2101 : : {
2102 [ + - + - : 8 : r = handle_buildid (0, buildid, "debuginfo", "", &alt_fd);
+ - ]
2103 : : }
2104 [ - - ]: 0 : catch (const reportable_exception& e)
2105 : : {
2106 : : // swallow exceptions
2107 : : }
2108 : :
2109 : : // NB: this is not actually recursive! This invokes the web-query
2110 : : // path, which cannot get back into the scan code paths.
2111 [ + - ]: 8 : if (r)
2112 : : {
2113 : : // Found it!
2114 : 8 : altdbg_fd = dup(alt_fd); // ok if this fails, downstream failures ok
2115 : 8 : alt = altdbg = dwarf_begin (altdbg_fd, DWARF_C_READ);
2116 : : // NB: must close this dwarf and this fd at the bottom of the function!
2117 : 8 : MHD_destroy_response (r); // will close alt_fd
2118 [ + - ]: 8 : if (alt)
2119 : 8 : dwarf_setalt (dbg, alt);
2120 : : }
2121 : : }
2122 : : else
2123 : : {
2124 : : // NB: dwarf_setalt(alt) inappropriate - already done!
2125 : : // NB: altdbg will stay 0 so nothing tries to redundantly dealloc.
2126 : : }
2127 : :
2128 [ + - ]: 8 : if (alt)
2129 : : {
2130 [ + - ]: 8 : if (verbose > 3)
2131 : 8 : obatched(clog) << "Resolved altdebug buildid=" << buildid << endl;
2132 : : }
2133 : : else // (alt == NULL) - signal possible presence of poor debuginfo
2134 : : {
2135 : 0 : debug_sourcefiles.insert("");
2136 [ # # ]: 0 : if (verbose > 3)
2137 : 0 : obatched(clog) << "Unresolved altdebug buildid=" << buildid << endl;
2138 : : }
2139 : : }
2140 : :
2141 : 23 : Dwarf_Off offset = 0;
2142 : : Dwarf_Off old_offset;
2143 : : size_t hsize;
2144 : :
2145 [ + + ]: 350 : while (dwarf_nextcu (dbg, old_offset = offset, &offset, &hsize, NULL, NULL, NULL) == 0)
2146 : : {
2147 : : Dwarf_Die cudie_mem;
2148 : 327 : Dwarf_Die *cudie = dwarf_offdie (dbg, old_offset + hsize, &cudie_mem);
2149 : :
2150 [ - + ]: 327 : if (cudie == NULL)
2151 : 4 : continue;
2152 [ + + ]: 327 : if (dwarf_tag (cudie) != DW_TAG_compile_unit)
2153 : 4 : continue;
2154 : :
2155 [ + - ]: 323 : const char *cuname = dwarf_diename(cudie) ?: "unknown";
2156 : :
2157 : : Dwarf_Files *files;
2158 : : size_t nfiles;
2159 [ - + ]: 323 : if (dwarf_getsrcfiles (cudie, &files, &nfiles) != 0)
2160 : 0 : continue;
2161 : :
2162 : : // extract DW_AT_comp_dir to resolve relative file names
2163 : 323 : const char *comp_dir = "";
2164 : : const char *const *dirs;
2165 : : size_t ndirs;
2166 [ + - + - ]: 646 : if (dwarf_getsrcdirs (files, &dirs, &ndirs) == 0 &&
2167 [ + - ]: 323 : dirs[0] != NULL)
2168 : 323 : comp_dir = dirs[0];
2169 [ - + ]: 323 : if (comp_dir == NULL)
2170 : 0 : comp_dir = "";
2171 : :
2172 [ + + ]: 323 : if (verbose > 3)
2173 : 34 : obatched(clog) << "searching for sources for cu=" << cuname << " comp_dir=" << comp_dir
2174 : 17 : << " #files=" << nfiles << " #dirs=" << ndirs << endl;
2175 : :
2176 [ - + - - ]: 323 : if (comp_dir[0] == '\0' && cuname[0] != '/')
2177 : : {
2178 : : // This is a common symptom for dwz-compressed debug files,
2179 : : // where the altdebug file cannot be resolved.
2180 [ # # ]: 0 : if (verbose > 3)
2181 : 0 : obatched(clog) << "skipping cu=" << cuname << " due to empty comp_dir" << endl;
2182 : 0 : continue;
2183 : : }
2184 : :
2185 [ + + ]: 5069 : for (size_t f = 1; f < nfiles; f++)
2186 : : {
2187 : 4746 : const char *hat = dwarf_filesrc (files, f, NULL, NULL);
2188 [ - + ]: 4746 : if (hat == NULL)
2189 : 0 : continue;
2190 : :
2191 [ - + ]: 4746 : if (string(hat) == "<built-in>") // gcc intrinsics, don't bother record
2192 : 0 : continue;
2193 : :
2194 : 4746 : string waldo;
2195 [ + + ]: 4746 : if (hat[0] == '/') // absolute
2196 : 2933 : waldo = (string (hat));
2197 [ + - ]: 1813 : else if (comp_dir[0] != '\0') // comp_dir relative
2198 : 1813 : waldo = (string (comp_dir) + string("/") + string (hat));
2199 : : else
2200 : : {
2201 [ # # ]: 0 : if (verbose > 3)
2202 : 0 : obatched(clog) << "skipping hat=" << hat << " due to empty comp_dir" << endl;
2203 : 0 : continue;
2204 : : }
2205 : :
2206 : : // NB: this is the 'waldo' that a dbginfo client will have
2207 : : // to supply for us to give them the file The comp_dir
2208 : : // prefixing is a definite complication. Otherwise we'd
2209 : : // have to return a setof comp_dirs (one per CU!) with
2210 : : // corresponding filesrc[] names, instead of one absolute
2211 : : // resoved set. Maybe we'll have to do that anyway. XXX
2212 : :
2213 [ - + ]: 4746 : if (verbose > 4)
2214 : 0 : obatched(clog) << waldo
2215 [ # # ]: 0 : << (debug_sourcefiles.find(waldo)==debug_sourcefiles.end() ? " new" : " dup") << endl;
2216 : :
2217 : 4746 : debug_sourcefiles.insert (waldo);
2218 : : }
2219 : : }
2220 : :
2221 : 23 : dwarf_end(dbg);
2222 [ + + ]: 23 : if (altdbg)
2223 : 8 : dwarf_end(altdbg);
2224 [ + + ]: 23 : if (altdbg_fd >= 0)
2225 : 8 : close(altdbg_fd);
2226 : : }
2227 : :
2228 : :
2229 : :
2230 : : static void
2231 : 112 : elf_classify (int fd, bool &executable_p, bool &debuginfo_p, string &buildid, set<string>& debug_sourcefiles)
2232 : : {
2233 : 112 : Elf *elf = elf_begin (fd, ELF_C_READ_MMAP_PRIVATE, NULL);
2234 [ - + ]: 114 : if (elf == NULL)
2235 : 0 : return;
2236 : :
2237 : : try // catch our types of errors and clean up the Elf* object
2238 : : {
2239 [ + - + + ]: 114 : if (elf_kind (elf) != ELF_K_ELF)
2240 : : {
2241 [ + - ]: 72 : elf_end (elf);
2242 : 72 : return;
2243 : : }
2244 : :
2245 : : GElf_Ehdr ehdr_storage;
2246 [ + - ]: 42 : GElf_Ehdr *ehdr = gelf_getehdr (elf, &ehdr_storage);
2247 [ - + ]: 42 : if (ehdr == NULL)
2248 : : {
2249 [ # # ]: 0 : elf_end (elf);
2250 : 0 : return;
2251 : : }
2252 : 42 : auto elf_type = ehdr->e_type;
2253 : :
2254 : : const void *build_id; // elfutils-owned memory
2255 [ + - ]: 42 : ssize_t sz = dwelf_elf_gnu_build_id (elf, & build_id);
2256 [ - + ]: 42 : if (sz <= 0)
2257 : : {
2258 : : // It's not a diagnostic-worthy error for an elf file to lack build-id.
2259 : : // It might just be very old.
2260 [ # # ]: 0 : elf_end (elf);
2261 : 0 : return;
2262 : : }
2263 : :
2264 : : // build_id is a raw byte array; convert to hexadecimal *lowercase*
2265 : 42 : unsigned char* build_id_bytes = (unsigned char*) build_id;
2266 [ + + ]: 878 : for (ssize_t idx=0; idx<sz; idx++)
2267 : : {
2268 [ + - ]: 836 : buildid += "0123456789abcdef"[build_id_bytes[idx] >> 4];
2269 [ + - ]: 833 : buildid += "0123456789abcdef"[build_id_bytes[idx] & 0xf];
2270 : : }
2271 : :
2272 : : // now decide whether it's an executable - namely, any allocatable section has
2273 : : // PROGBITS;
2274 [ + + + + ]: 42 : if (elf_type == ET_EXEC || elf_type == ET_DYN)
2275 : : {
2276 : : size_t shnum;
2277 [ + - ]: 36 : int rc = elf_getshdrnum (elf, &shnum);
2278 [ - + ]: 36 : if (rc < 0)
2279 [ # # # # ]: 0 : throw elfutils_exception(rc, "getshdrnum");
2280 : :
2281 : 36 : executable_p = false;
2282 [ + + ]: 680 : for (size_t sc = 0; sc < shnum; sc++)
2283 : : {
2284 [ + - ]: 661 : Elf_Scn *scn = elf_getscn (elf, sc);
2285 [ - + ]: 661 : if (scn == NULL)
2286 : 0 : continue;
2287 : :
2288 : : GElf_Shdr shdr_mem;
2289 [ + - ]: 661 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_mem);
2290 [ - + ]: 663 : if (shdr == NULL)
2291 : 0 : continue;
2292 : :
2293 : : // allocated (loadable / vm-addr-assigned) section with available content?
2294 [ + + + + ]: 663 : if ((shdr->sh_type == SHT_PROGBITS) && (shdr->sh_flags & SHF_ALLOC))
2295 : : {
2296 [ - + ]: 19 : if (verbose > 4)
2297 [ # # # # : 0 : obatched(clog) << "executable due to SHF_ALLOC SHT_PROGBITS sc=" << sc << endl;
# # # # ]
2298 : 19 : executable_p = true;
2299 : 19 : break; // no need to keep looking for others
2300 : : }
2301 : : } // iterate over sections
2302 : : } // executable_p classification
2303 : :
2304 : : // now decide whether it's a debuginfo - namely, if it has any .debug* or .zdebug* sections
2305 : : // logic mostly stolen from fweimer@redhat.com's elfclassify drafts
2306 : : size_t shstrndx;
2307 [ + - ]: 44 : int rc = elf_getshdrstrndx (elf, &shstrndx);
2308 [ - + ]: 42 : if (rc < 0)
2309 [ # # # # ]: 0 : throw elfutils_exception(rc, "getshdrstrndx");
2310 : :
2311 : 42 : Elf_Scn *scn = NULL;
2312 : 42 : bool symtab_p = false;
2313 : 42 : bool bits_alloc_p = false;
2314 : : while (true)
2315 : : {
2316 [ + - ]: 1075 : scn = elf_nextscn (elf, scn);
2317 [ + + ]: 1077 : if (scn == NULL)
2318 : 19 : break;
2319 : : GElf_Shdr shdr_storage;
2320 [ + - ]: 1058 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_storage);
2321 [ - + ]: 1057 : if (shdr == NULL)
2322 : 0 : break;
2323 [ + - ]: 1057 : const char *section_name = elf_strptr (elf, shstrndx, shdr->sh_name);
2324 [ - + ]: 1056 : if (section_name == NULL)
2325 : 0 : break;
2326 [ + + ]: 1056 : if (strncmp(section_name, ".debug_line", 11) == 0 ||
2327 [ - + ]: 1033 : strncmp(section_name, ".zdebug_line", 12) == 0)
2328 : : {
2329 : 23 : debuginfo_p = true;
2330 : 23 : dwarf_extract_source_paths (elf, debug_sourcefiles);
2331 : 23 : break; // expecting only one .*debug_line, so no need to look for others
2332 : : }
2333 [ + + ]: 1033 : else if (strncmp(section_name, ".debug_", 7) == 0 ||
2334 [ # + ]: 965 : strncmp(section_name, ".zdebug_", 8) == 0)
2335 : : {
2336 : 67 : debuginfo_p = true;
2337 : : // NB: don't break; need to parse .debug_line for sources
2338 : : }
2339 [ + + ]: 966 : else if (shdr->sh_type == SHT_SYMTAB)
2340 : : {
2341 : 1 : symtab_p = true;
2342 : : }
2343 [ + + ]: 965 : else if (shdr->sh_type != SHT_NOBITS
2344 [ + + ]: 563 : && shdr->sh_type != SHT_NOTE
2345 [ + + ]: 477 : && (shdr->sh_flags & SHF_ALLOC) != 0)
2346 : : {
2347 : 408 : bits_alloc_p = true;
2348 : : }
2349 : 1033 : }
2350 : :
2351 : : // For more expansive elf/split-debuginfo classification, we
2352 : : // want to identify as debuginfo "strip -s"-produced files
2353 : : // without .debug_info* (like libicudata), but we don't want to
2354 : : // identify "strip -g" executables (with .symtab left there).
2355 [ + + - + ]: 42 : if (symtab_p && !bits_alloc_p)
2356 : 0 : debuginfo_p = true;
2357 : : }
2358 [ # # ]: 0 : catch (const reportable_exception& e)
2359 : : {
2360 [ # # ]: 0 : e.report(clog);
2361 : : }
2362 : 42 : elf_end (elf);
2363 : : }
2364 : :
2365 : :
2366 : : static void
2367 : 173 : scan_source_file (const string& rps, const stat_t& st,
2368 : : sqlite_ps& ps_upsert_buildids,
2369 : : sqlite_ps& ps_upsert_files,
2370 : : sqlite_ps& ps_upsert_de,
2371 : : sqlite_ps& ps_upsert_s,
2372 : : sqlite_ps& ps_query,
2373 : : sqlite_ps& ps_scan_done,
2374 : : unsigned& fts_cached,
2375 : : unsigned& fts_executable,
2376 : : unsigned& fts_debuginfo,
2377 : : unsigned& fts_sourcefiles)
2378 : : {
2379 : : /* See if we know of it already. */
2380 : : int rc = ps_query
2381 [ + - ]: 173 : .reset()
2382 [ + - ]: 173 : .bind(1, rps)
2383 [ + - ]: 173 : .bind(2, st.st_mtime)
2384 [ + - ]: 173 : .step();
2385 [ + - ]: 173 : ps_query.reset();
2386 [ + + ]: 173 : if (rc == SQLITE_ROW) // i.e., a result, as opposed to DONE (no results)
2387 : : // no need to recheck a file/version we already know
2388 : : // specifically, no need to elf-begin a file we already determined is non-elf
2389 : : // (so is stored with buildid=NULL)
2390 : : {
2391 : 128 : fts_cached++;
2392 : 128 : return;
2393 : : }
2394 : :
2395 : 45 : bool executable_p = false, debuginfo_p = false; // E and/or D
2396 : 90 : string buildid;
2397 : 90 : set<string> sourcefiles;
2398 : :
2399 [ + - ]: 45 : int fd = open (rps.c_str(), O_RDONLY);
2400 : : try
2401 : : {
2402 [ + - ]: 45 : if (fd >= 0)
2403 [ + - ]: 45 : elf_classify (fd, executable_p, debuginfo_p, buildid, sourcefiles);
2404 : : else
2405 [ # # # # : 0 : throw libc_exception(errno, string("open ") + rps);
# # ]
2406 : 45 : add_metric ("scanned_bytes_total","source","file",
2407 [ + - + - : 45 : st.st_size);
+ - + - ]
2408 [ + - + - : 45 : inc_metric ("scanned_files_total","source","file");
+ - + - ]
2409 : : }
2410 : : // NB: we catch exceptions here too, so that we can
2411 : : // cache the corrupt-elf case (!executable_p &&
2412 : : // !debuginfo_p) just below, just as if we had an
2413 : : // EPERM error from open(2).
2414 [ - - ]: 0 : catch (const reportable_exception& e)
2415 : : {
2416 [ - - ]: 0 : e.report(clog);
2417 : : }
2418 : :
2419 [ + - ]: 45 : if (fd >= 0)
2420 [ + - ]: 45 : close (fd);
2421 : :
2422 : : // register this file name in the interning table
2423 : : ps_upsert_files
2424 [ + - ]: 45 : .reset()
2425 [ + - ]: 45 : .bind(1, rps)
2426 [ + - ]: 45 : .step_ok_done();
2427 : :
2428 [ + + ]: 45 : if (buildid == "")
2429 : : {
2430 : : // no point storing an elf file without buildid
2431 : 41 : executable_p = false;
2432 : 41 : debuginfo_p = false;
2433 : : }
2434 : : else
2435 : : {
2436 : : // register this build-id in the interning table
2437 : : ps_upsert_buildids
2438 [ + - ]: 4 : .reset()
2439 [ + - ]: 4 : .bind(1, buildid)
2440 [ + - ]: 4 : .step_ok_done();
2441 : : }
2442 : :
2443 [ + + ]: 45 : if (executable_p)
2444 : 3 : fts_executable ++;
2445 [ + + ]: 45 : if (debuginfo_p)
2446 : 3 : fts_debuginfo ++;
2447 [ + + + + ]: 45 : if (executable_p || debuginfo_p)
2448 : : {
2449 : : ps_upsert_de
2450 [ + - ]: 4 : .reset()
2451 [ + - ]: 4 : .bind(1, buildid)
2452 [ + + + - ]: 4 : .bind(2, debuginfo_p ? 1 : 0)
2453 [ + + + - ]: 4 : .bind(3, executable_p ? 1 : 0)
2454 [ + - ]: 4 : .bind(4, rps)
2455 [ + - ]: 4 : .bind(5, st.st_mtime)
2456 [ + - ]: 4 : .step_ok_done();
2457 : : }
2458 [ + + ]: 45 : if (executable_p)
2459 [ + - + - : 3 : inc_metric("found_executable_total","source","files");
+ - + - ]
2460 [ + + ]: 45 : if (debuginfo_p)
2461 [ + - + - : 3 : inc_metric("found_debuginfo_total","source","files");
+ - + - ]
2462 : :
2463 [ + + + - : 45 : if (sourcefiles.size() && buildid != "")
+ + ]
2464 : : {
2465 : 3 : fts_sourcefiles += sourcefiles.size();
2466 : :
2467 [ + + ]: 484 : for (auto&& dwarfsrc : sourcefiles)
2468 : : {
2469 : 481 : char *srp = realpath(dwarfsrc.c_str(), NULL);
2470 [ + + ]: 481 : if (srp == NULL) // also if DWZ unresolved dwarfsrc=""
2471 : 6 : continue; // unresolvable files are not a serious problem
2472 : : // throw libc_exception(errno, "fts/file realpath " + srcpath);
2473 [ + - ]: 475 : string srps = string(srp);
2474 : 475 : free (srp);
2475 : :
2476 : : struct stat sfs;
2477 : 475 : rc = stat(srps.c_str(), &sfs);
2478 [ - + ]: 475 : if (rc != 0)
2479 : 0 : continue;
2480 : :
2481 [ + - ]: 475 : if (verbose > 2)
2482 [ + - + - ]: 950 : obatched(clog) << "recorded buildid=" << buildid << " file=" << srps
2483 [ + - + - : 475 : << " mtime=" << sfs.st_mtime
+ - + - +
- ]
2484 [ + - + - : 475 : << " as source " << dwarfsrc << endl;
+ - ]
2485 : :
2486 : : ps_upsert_files
2487 [ + - ]: 475 : .reset()
2488 [ + - ]: 475 : .bind(1, srps)
2489 [ + - ]: 475 : .step_ok_done();
2490 : :
2491 : : // PR25548: store canonicalized dwarfsrc path
2492 [ + - ]: 950 : string dwarfsrc_canon = canon_pathname (dwarfsrc);
2493 [ + + ]: 475 : if (dwarfsrc_canon != dwarfsrc)
2494 : : {
2495 [ + + ]: 85 : if (verbose > 3)
2496 [ + - + - : 1 : obatched(clog) << "canonicalized src=" << dwarfsrc << " alias=" << dwarfsrc_canon << endl;
+ - + - +
- + - ]
2497 : : }
2498 : :
2499 : : ps_upsert_files
2500 [ + - ]: 475 : .reset()
2501 [ + - ]: 475 : .bind(1, dwarfsrc_canon)
2502 [ + - ]: 475 : .step_ok_done();
2503 : :
2504 : : ps_upsert_s
2505 [ + - ]: 475 : .reset()
2506 [ + - ]: 475 : .bind(1, buildid)
2507 [ + - ]: 475 : .bind(2, dwarfsrc_canon)
2508 [ + - ]: 475 : .bind(3, srps)
2509 [ + - ]: 475 : .bind(4, sfs.st_mtime)
2510 [ + - ]: 475 : .step_ok_done();
2511 : :
2512 [ + - + - : 475 : inc_metric("found_sourcerefs_total","source","files");
+ - + - ]
2513 : : }
2514 : : }
2515 : :
2516 : : ps_scan_done
2517 [ + - ]: 45 : .reset()
2518 [ + - ]: 45 : .bind(1, rps)
2519 [ + - ]: 45 : .bind(2, st.st_mtime)
2520 [ + - ]: 45 : .bind(3, st.st_size)
2521 [ + - ]: 45 : .step_ok_done();
2522 : :
2523 [ + - ]: 45 : if (verbose > 2)
2524 [ + - + - ]: 90 : obatched(clog) << "recorded buildid=" << buildid << " file=" << rps
2525 [ + - + - : 45 : << " mtime=" << st.st_mtime << " atype="
+ - + - +
- ]
2526 : : << (executable_p ? "E" : "")
2527 [ + - + + : 45 : << (debuginfo_p ? "D" : "") << endl;
+ - + + +
- + - ]
2528 : : }
2529 : :
2530 : :
2531 : :
2532 : :
2533 : :
2534 : : // Analyze given archive file of given age; record buildids / exec/debuginfo-ness of its
2535 : : // constituent files with given upsert statements.
2536 : : static void
2537 : 36 : archive_classify (const string& rps, string& archive_extension,
2538 : : sqlite_ps& ps_upsert_buildids, sqlite_ps& ps_upsert_files,
2539 : : sqlite_ps& ps_upsert_de, sqlite_ps& ps_upsert_sref, sqlite_ps& ps_upsert_sdef,
2540 : : time_t mtime,
2541 : : unsigned& fts_executable, unsigned& fts_debuginfo, unsigned& fts_sref, unsigned& fts_sdef,
2542 : : bool& fts_sref_complete_p)
2543 : : {
2544 [ + - ]: 72 : string archive_decoder = "/dev/null";
2545 [ + + ]: 144 : for (auto&& arch : scan_archives)
2546 [ + - + + ]: 108 : if (string_endswith(rps, arch.first))
2547 : : {
2548 [ + - ]: 36 : archive_extension = arch.first;
2549 [ + - ]: 36 : archive_decoder = arch.second;
2550 : : }
2551 : :
2552 : : FILE* fp;
2553 : : defer_dtor<FILE*,int>::dtor_fn dfn;
2554 [ + + ]: 36 : if (archive_decoder != "cat")
2555 : : {
2556 [ + - + - : 9 : string popen_cmd = archive_decoder + " " + shell_escape(rps);
+ - ]
2557 [ + - ]: 3 : fp = popen (popen_cmd.c_str(), "r"); // "e" O_CLOEXEC?
2558 : 3 : dfn = pclose;
2559 [ - + ]: 3 : if (fp == NULL)
2560 [ # # # # : 0 : throw libc_exception (errno, string("popen ") + popen_cmd);
# # ]
2561 : : }
2562 : : else
2563 : : {
2564 [ + - ]: 33 : fp = fopen (rps.c_str(), "r");
2565 : 33 : dfn = fclose;
2566 [ - + ]: 33 : if (fp == NULL)
2567 [ # # # # : 0 : throw libc_exception (errno, string("fopen ") + rps);
# # ]
2568 : : }
2569 : 72 : defer_dtor<FILE*,int> fp_closer (fp, dfn);
2570 : :
2571 : : struct archive *a;
2572 [ + - ]: 36 : a = archive_read_new();
2573 [ - + ]: 36 : if (a == NULL)
2574 [ # # # # ]: 0 : throw archive_exception("cannot create archive reader");
2575 : 72 : defer_dtor<struct archive*,int> archive_closer (a, archive_read_free);
2576 : :
2577 [ + - ]: 36 : int rc = archive_read_support_format_all(a);
2578 [ - + ]: 36 : if (rc != ARCHIVE_OK)
2579 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all formats");
2580 [ + - ]: 36 : rc = archive_read_support_filter_all(a);
2581 [ - + ]: 36 : if (rc != ARCHIVE_OK)
2582 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all filters");
2583 : :
2584 [ + - ]: 36 : rc = archive_read_open_FILE (a, fp);
2585 [ - + ]: 36 : if (rc != ARCHIVE_OK)
2586 [ # # # # ]: 0 : throw archive_exception(a, "cannot open archive from pipe");
2587 : :
2588 [ + + ]: 36 : if (verbose > 3)
2589 [ + - + - : 34 : obatched(clog) << "libarchive scanning " << rps << endl;
+ - + - ]
2590 : :
2591 : : while(1) // parse archive entries
2592 : : {
2593 [ - + ]: 255 : if (interrupted)
2594 : 0 : break;
2595 : :
2596 : : try
2597 : : {
2598 : : struct archive_entry *e;
2599 [ + - ]: 255 : rc = archive_read_next_header (a, &e);
2600 [ + + ]: 255 : if (rc != ARCHIVE_OK)
2601 : 36 : break;
2602 : :
2603 [ + - + + ]: 219 : if (! S_ISREG(archive_entry_mode (e))) // skip non-files completely
2604 : 150 : continue;
2605 : :
2606 [ + - ]: 138 : string fn = canonicalized_archive_entry_pathname (e);
2607 : :
2608 [ + + ]: 69 : if (verbose > 3)
2609 [ + - + - : 64 : obatched(clog) << "libarchive checking " << fn << endl;
+ - + - ]
2610 : :
2611 : : // extract this file to a temporary file
2612 : 69 : char* tmppath = NULL;
2613 : 69 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
2614 [ - + ]: 69 : if (rc < 0)
2615 [ # # # # ]: 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
2616 : 138 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
2617 [ + - ]: 69 : int fd = mkstemp (tmppath);
2618 [ - + ]: 69 : if (fd < 0)
2619 [ # # # # ]: 0 : throw libc_exception (errno, "cannot create temporary file");
2620 : 69 : unlink (tmppath); // unlink now so OS will release the file as soon as we close the fd
2621 : 138 : defer_dtor<int,int> minifd_closer (fd, close);
2622 : :
2623 [ + - ]: 69 : rc = archive_read_data_into_fd (a, fd);
2624 [ - + ]: 69 : if (rc != ARCHIVE_OK)
2625 [ # # # # ]: 0 : throw archive_exception(a, "cannot extract file");
2626 : :
2627 : : // finally ... time to run elf_classify on this bad boy and update the database
2628 : 69 : bool executable_p = false, debuginfo_p = false;
2629 : 138 : string buildid;
2630 : 138 : set<string> sourcefiles;
2631 [ + - ]: 69 : elf_classify (fd, executable_p, debuginfo_p, buildid, sourcefiles);
2632 : : // NB: might throw
2633 : :
2634 [ + + ]: 69 : if (buildid != "") // intern buildid
2635 : : {
2636 : : ps_upsert_buildids
2637 [ + - ]: 38 : .reset()
2638 [ + - ]: 38 : .bind(1, buildid)
2639 [ + - ]: 38 : .step_ok_done();
2640 : : }
2641 : :
2642 : : ps_upsert_files // register this rpm constituent file name in interning table
2643 [ + - ]: 69 : .reset()
2644 [ + - ]: 69 : .bind(1, fn)
2645 [ + - ]: 69 : .step_ok_done();
2646 : :
2647 [ + + ]: 69 : if (sourcefiles.size() > 0) // sref records needed
2648 : : {
2649 : : // NB: we intern each source file once. Once raw, as it
2650 : : // appears in the DWARF file list coming back from
2651 : : // elf_classify() - because it'll end up in the
2652 : : // _norm.artifactsrc column. We don't also put another
2653 : : // version with a '.' at the front, even though that's
2654 : : // how rpm/cpio packs names, because we hide that from
2655 : : // the database for storage efficiency.
2656 : :
2657 [ + + ]: 61 : for (auto&& s : sourcefiles)
2658 : : {
2659 [ - + ]: 45 : if (s == "")
2660 : : {
2661 : 0 : fts_sref_complete_p = false;
2662 : 0 : continue;
2663 : : }
2664 : :
2665 : : // PR25548: store canonicalized source path
2666 : 45 : const string& dwarfsrc = s;
2667 [ + - ]: 45 : string dwarfsrc_canon = canon_pathname (dwarfsrc);
2668 [ + + ]: 45 : if (dwarfsrc_canon != dwarfsrc)
2669 : : {
2670 [ + - ]: 2 : if (verbose > 3)
2671 [ + - + - : 2 : obatched(clog) << "canonicalized src=" << dwarfsrc << " alias=" << dwarfsrc_canon << endl;
+ - + - +
- + - ]
2672 : : }
2673 : :
2674 : : ps_upsert_files
2675 [ + - ]: 45 : .reset()
2676 [ + - ]: 45 : .bind(1, dwarfsrc_canon)
2677 [ + - ]: 45 : .step_ok_done();
2678 : :
2679 : : ps_upsert_sref
2680 [ + - ]: 45 : .reset()
2681 [ + - ]: 45 : .bind(1, buildid)
2682 [ + - ]: 45 : .bind(2, dwarfsrc_canon)
2683 [ + - ]: 45 : .step_ok_done();
2684 : :
2685 : 45 : fts_sref ++;
2686 : : }
2687 : : }
2688 : :
2689 [ + + ]: 69 : if (executable_p)
2690 : 16 : fts_executable ++;
2691 [ + + ]: 69 : if (debuginfo_p)
2692 : 22 : fts_debuginfo ++;
2693 : :
2694 [ + + + + ]: 69 : if (executable_p || debuginfo_p)
2695 : : {
2696 : : ps_upsert_de
2697 [ + - ]: 38 : .reset()
2698 [ + - ]: 38 : .bind(1, buildid)
2699 [ + + + - ]: 38 : .bind(2, debuginfo_p ? 1 : 0)
2700 [ + + + - ]: 38 : .bind(3, executable_p ? 1 : 0)
2701 [ + - ]: 38 : .bind(4, rps)
2702 [ + - ]: 38 : .bind(5, mtime)
2703 [ + - ]: 38 : .bind(6, fn)
2704 [ + - ]: 38 : .step_ok_done();
2705 : : }
2706 : : else // potential source - sdef record
2707 : : {
2708 : 31 : fts_sdef ++;
2709 : : ps_upsert_sdef
2710 [ + - ]: 31 : .reset()
2711 [ + - ]: 31 : .bind(1, rps)
2712 [ + - ]: 31 : .bind(2, mtime)
2713 [ + - ]: 31 : .bind(3, fn)
2714 [ + - ]: 31 : .step_ok_done();
2715 : : }
2716 : :
2717 [ + - + + : 69 : if ((verbose > 2) && (executable_p || debuginfo_p))
+ + ]
2718 [ + - + - ]: 76 : obatched(clog) << "recorded buildid=" << buildid << " rpm=" << rps << " file=" << fn
2719 [ + - + - : 38 : << " mtime=" << mtime << " atype="
+ - + - +
- + - +
- ]
2720 : : << (executable_p ? "E" : "")
2721 : : << (debuginfo_p ? "D" : "")
2722 [ + - + + : 38 : << " sourcefiles=" << sourcefiles.size() << endl;
+ - + + +
- + - + -
+ - ]
2723 : :
2724 : : }
2725 [ - - ]: 0 : catch (const reportable_exception& e)
2726 : : {
2727 [ - - ]: 0 : e.report(clog);
2728 : : }
2729 : 219 : }
2730 : 36 : }
2731 : :
2732 : :
2733 : :
2734 : : // scan for archive files such as .rpm
2735 : : static void
2736 : 118 : scan_archive_file (const string& rps, const stat_t& st,
2737 : : sqlite_ps& ps_upsert_buildids,
2738 : : sqlite_ps& ps_upsert_files,
2739 : : sqlite_ps& ps_upsert_de,
2740 : : sqlite_ps& ps_upsert_sref,
2741 : : sqlite_ps& ps_upsert_sdef,
2742 : : sqlite_ps& ps_query,
2743 : : sqlite_ps& ps_scan_done,
2744 : : unsigned& fts_cached,
2745 : : unsigned& fts_executable,
2746 : : unsigned& fts_debuginfo,
2747 : : unsigned& fts_sref,
2748 : : unsigned& fts_sdef)
2749 : : {
2750 : : /* See if we know of it already. */
2751 : : int rc = ps_query
2752 [ + - ]: 118 : .reset()
2753 [ + - ]: 118 : .bind(1, rps)
2754 [ + - ]: 118 : .bind(2, st.st_mtime)
2755 [ + - ]: 118 : .step();
2756 [ + - ]: 118 : ps_query.reset();
2757 [ + + ]: 118 : if (rc == SQLITE_ROW) // i.e., a result, as opposed to DONE (no results)
2758 : : // no need to recheck a file/version we already know
2759 : : // specifically, no need to parse this archive again, since we already have
2760 : : // it as a D or E or S record,
2761 : : // (so is stored with buildid=NULL)
2762 : : {
2763 : 82 : fts_cached ++;
2764 : 82 : return;
2765 : : }
2766 : :
2767 : : // intern the archive file name
2768 : : ps_upsert_files
2769 [ + - ]: 36 : .reset()
2770 [ + - ]: 36 : .bind(1, rps)
2771 [ + - ]: 36 : .step_ok_done();
2772 : :
2773 : : // extract the archive contents
2774 : 36 : unsigned my_fts_executable = 0, my_fts_debuginfo = 0, my_fts_sref = 0, my_fts_sdef = 0;
2775 : 36 : bool my_fts_sref_complete_p = true;
2776 : : try
2777 : : {
2778 : 72 : string archive_extension;
2779 : 36 : archive_classify (rps, archive_extension,
2780 : : ps_upsert_buildids, ps_upsert_files,
2781 : : ps_upsert_de, ps_upsert_sref, ps_upsert_sdef, // dalt
2782 [ + - ]: 36 : st.st_mtime,
2783 : : my_fts_executable, my_fts_debuginfo, my_fts_sref, my_fts_sdef,
2784 : : my_fts_sref_complete_p);
2785 [ + - ]: 72 : add_metric ("scanned_bytes_total","source",archive_extension + " archive",
2786 [ + - + - : 108 : st.st_size);
+ - ]
2787 [ + - + - : 36 : inc_metric ("scanned_files_total","source",archive_extension + " archive");
+ - + - ]
2788 [ + - + - ]: 36 : add_metric("found_debuginfo_total","source",archive_extension + " archive",
2789 [ + - + - ]: 72 : my_fts_debuginfo);
2790 [ + - + - ]: 36 : add_metric("found_executable_total","source",archive_extension + " archive",
2791 [ + - + - ]: 72 : my_fts_executable);
2792 [ + - + - ]: 36 : add_metric("found_sourcerefs_total","source",archive_extension + " archive",
2793 [ + - + - ]: 72 : my_fts_sref);
2794 : : }
2795 [ - - ]: 0 : catch (const reportable_exception& e)
2796 : : {
2797 [ - - ]: 0 : e.report(clog);
2798 : : }
2799 : :
2800 [ + - ]: 36 : if (verbose > 2)
2801 [ + - + - ]: 72 : obatched(clog) << "scanned archive=" << rps
2802 [ + - + - : 36 : << " mtime=" << st.st_mtime
+ - ]
2803 [ + - ]: 36 : << " executables=" << my_fts_executable
2804 [ + - + - ]: 36 : << " debuginfos=" << my_fts_debuginfo
2805 [ + - + - ]: 36 : << " srefs=" << my_fts_sref
2806 [ + - + - ]: 36 : << " sdefs=" << my_fts_sdef
2807 [ + - + - ]: 36 : << endl;
2808 : :
2809 : 36 : fts_executable += my_fts_executable;
2810 : 36 : fts_debuginfo += my_fts_debuginfo;
2811 : 36 : fts_sref += my_fts_sref;
2812 : 36 : fts_sdef += my_fts_sdef;
2813 : :
2814 [ + - ]: 36 : if (my_fts_sref_complete_p) // leave incomplete?
2815 : : ps_scan_done
2816 [ + - ]: 36 : .reset()
2817 [ + - ]: 36 : .bind(1, rps)
2818 [ + - ]: 36 : .bind(2, st.st_mtime)
2819 [ + - ]: 36 : .bind(3, st.st_size)
2820 [ + - ]: 36 : .step_ok_done();
2821 : : }
2822 : :
2823 : :
2824 : :
2825 : : ////////////////////////////////////////////////////////////////////////
2826 : :
2827 : :
2828 : :
2829 : : // The thread that consumes file names off of the scanq. We hold
2830 : : // the persistent sqlite_ps's at this level and delegate file/archive
2831 : : // scanning to other functions.
2832 : : static void*
2833 : 12 : thread_main_scanner (void* arg)
2834 : : {
2835 : : (void) arg;
2836 : :
2837 : : // all the prepared statements fit to use, the _f_ set:
2838 [ + - + - : 48 : sqlite_ps ps_f_upsert_buildids (db, "file-buildids-intern", "insert or ignore into " BUILDIDS "_buildids VALUES (NULL, ?);");
+ - ]
2839 [ + - + - : 48 : sqlite_ps ps_f_upsert_files (db, "file-files-intern", "insert or ignore into " BUILDIDS "_files VALUES (NULL, ?);");
+ - ]
2840 : : sqlite_ps ps_f_upsert_de (db, "file-de-upsert",
2841 : : "insert or ignore into " BUILDIDS "_f_de "
2842 : : "(buildid, debuginfo_p, executable_p, file, mtime) "
2843 : : "values ((select id from " BUILDIDS "_buildids where hex = ?),"
2844 : : " ?,?,"
2845 [ + - + - : 48 : " (select id from " BUILDIDS "_files where name = ?), ?);");
+ - ]
2846 : : sqlite_ps ps_f_upsert_s (db, "file-s-upsert",
2847 : : "insert or ignore into " BUILDIDS "_f_s "
2848 : : "(buildid, artifactsrc, file, mtime) "
2849 : : "values ((select id from " BUILDIDS "_buildids where hex = ?),"
2850 : : " (select id from " BUILDIDS "_files where name = ?),"
2851 : : " (select id from " BUILDIDS "_files where name = ?),"
2852 [ + - + - : 48 : " ?);");
+ - ]
2853 : : sqlite_ps ps_f_query (db, "file-negativehit-find",
2854 : : "select 1 from " BUILDIDS "_file_mtime_scanned where sourcetype = 'F' "
2855 [ + - + - : 48 : "and file = (select id from " BUILDIDS "_files where name = ?) and mtime = ?;");
+ - ]
2856 : : sqlite_ps ps_f_scan_done (db, "file-scanned",
2857 : : "insert or ignore into " BUILDIDS "_file_mtime_scanned (sourcetype, file, mtime, size)"
2858 [ + - + - : 48 : "values ('F', (select id from " BUILDIDS "_files where name = ?), ?, ?);");
+ - ]
2859 : :
2860 : : // and now for the _r_ set
2861 [ + - + - : 48 : sqlite_ps ps_r_upsert_buildids (db, "rpm-buildid-intern", "insert or ignore into " BUILDIDS "_buildids VALUES (NULL, ?);");
+ - ]
2862 [ + - + - : 48 : sqlite_ps ps_r_upsert_files (db, "rpm-file-intern", "insert or ignore into " BUILDIDS "_files VALUES (NULL, ?);");
+ - ]
2863 : : sqlite_ps ps_r_upsert_de (db, "rpm-de-insert",
2864 : : "insert or ignore into " BUILDIDS "_r_de (buildid, debuginfo_p, executable_p, file, mtime, content) values ("
2865 : : "(select id from " BUILDIDS "_buildids where hex = ?), ?, ?, "
2866 : : "(select id from " BUILDIDS "_files where name = ?), ?, "
2867 [ + - + - : 48 : "(select id from " BUILDIDS "_files where name = ?));");
+ - ]
2868 : : sqlite_ps ps_r_upsert_sref (db, "rpm-sref-insert",
2869 : : "insert or ignore into " BUILDIDS "_r_sref (buildid, artifactsrc) values ("
2870 : : "(select id from " BUILDIDS "_buildids where hex = ?), "
2871 [ + - + - : 48 : "(select id from " BUILDIDS "_files where name = ?));");
+ - ]
2872 : : sqlite_ps ps_r_upsert_sdef (db, "rpm-sdef-insert",
2873 : : "insert or ignore into " BUILDIDS "_r_sdef (file, mtime, content) values ("
2874 : : "(select id from " BUILDIDS "_files where name = ?), ?,"
2875 [ + - + - : 48 : "(select id from " BUILDIDS "_files where name = ?));");
+ - ]
2876 : : sqlite_ps ps_r_query (db, "rpm-negativehit-query",
2877 : : "select 1 from " BUILDIDS "_file_mtime_scanned where "
2878 [ + - + - : 48 : "sourcetype = 'R' and file = (select id from " BUILDIDS "_files where name = ?) and mtime = ?;");
+ - ]
2879 : : sqlite_ps ps_r_scan_done (db, "rpm-scanned",
2880 : : "insert or ignore into " BUILDIDS "_file_mtime_scanned (sourcetype, file, mtime, size)"
2881 [ + - + - : 36 : "values ('R', (select id from " BUILDIDS "_files where name = ?), ?, ?);");
+ - ]
2882 : :
2883 : :
2884 : 12 : unsigned fts_cached = 0, fts_executable = 0, fts_debuginfo = 0, fts_sourcefiles = 0;
2885 : 12 : unsigned fts_sref = 0, fts_sdef = 0;
2886 : :
2887 [ + - + - : 12 : add_metric("thread_count", "role", "scan", 1);
+ - + - ]
2888 [ + - + - : 12 : add_metric("thread_busy", "role", "scan", 1);
+ - + - ]
2889 [ + + ]: 197 : while (! interrupted)
2890 : : {
2891 : 185 : scan_payload p;
2892 : :
2893 [ + - + - : 185 : add_metric("thread_busy", "role", "scan", -1);
+ - + - ]
2894 [ + - ]: 185 : bool gotone = scanq.wait_front(p);
2895 [ + - + - : 185 : add_metric("thread_busy", "role", "scan", 1);
+ - + - ]
2896 : :
2897 [ + + ]: 185 : if (! gotone) continue; // go back to waiting
2898 : :
2899 : : try
2900 : : {
2901 : 173 : bool scan_archive = false;
2902 [ + + ]: 692 : for (auto&& arch : scan_archives)
2903 [ + - + + ]: 519 : if (string_endswith(p.first, arch.first))
2904 : 118 : scan_archive = true;
2905 : :
2906 [ + + ]: 173 : if (scan_archive)
2907 [ + - ]: 118 : scan_archive_file (p.first, p.second,
2908 : : ps_r_upsert_buildids,
2909 : : ps_r_upsert_files,
2910 : : ps_r_upsert_de,
2911 : : ps_r_upsert_sref,
2912 : : ps_r_upsert_sdef,
2913 : : ps_r_query,
2914 : : ps_r_scan_done,
2915 : : fts_cached,
2916 : : fts_executable,
2917 : : fts_debuginfo,
2918 : : fts_sref,
2919 : : fts_sdef);
2920 : :
2921 [ + - ]: 173 : if (scan_files) // NB: maybe "else if" ?
2922 [ + - ]: 173 : scan_source_file (p.first, p.second,
2923 : : ps_f_upsert_buildids,
2924 : : ps_f_upsert_files,
2925 : : ps_f_upsert_de,
2926 : : ps_f_upsert_s,
2927 : : ps_f_query,
2928 : : ps_f_scan_done,
2929 : : fts_cached, fts_executable, fts_debuginfo, fts_sourcefiles);
2930 : : }
2931 [ - - ]: 0 : catch (const reportable_exception& e)
2932 : : {
2933 [ - - ]: 0 : e.report(cerr);
2934 : : }
2935 : :
2936 [ + + + + : 173 : if (fts_cached || fts_executable || fts_debuginfo || fts_sourcefiles || fts_sref || fts_sdef)
- + - - -
- ]
2937 : : {} // NB: not just if a successful scan - we might have encountered -ENOSPC & failed
2938 [ + - + - ]: 173 : (void) statfs_free_enough_p(db_path, "database"); // report sqlite filesystem size
2939 [ + - + - ]: 173 : (void) statfs_free_enough_p(tmpdir, "tmpdir"); // this too, in case of fdcache/tmpfile usage
2940 : :
2941 : : // finished a scanning step -- not a "loop", because we just
2942 : : // consume the traversal loop's work, whenever
2943 [ + - + - : 173 : inc_metric("thread_work_total","role","scan");
+ - + - ]
2944 : : }
2945 : :
2946 : :
2947 [ + - + - : 12 : add_metric("thread_busy", "role", "scan", -1);
+ - + - ]
2948 : 24 : return 0;
2949 : : }
2950 : :
2951 : :
2952 : :
2953 : : // The thread that traverses all the source_paths and enqueues all the
2954 : : // matching files into the file/archive scan queue.
2955 : : static void
2956 : 12 : scan_source_paths()
2957 : : {
2958 : : // NB: fedora 31 glibc/fts(3) crashes inside fts_read() on empty
2959 : : // path list.
2960 [ + + ]: 12 : if (source_paths.empty())
2961 : 1 : return;
2962 : :
2963 : : // Turn the source_paths into an fts(3)-compatible char**. Since
2964 : : // source_paths[] does not change after argv processing, the
2965 : : // c_str()'s are safe to keep around awile.
2966 : 22 : vector<const char *> sps;
2967 [ + + ]: 51 : for (auto&& sp: source_paths)
2968 [ + - ]: 40 : sps.push_back(sp.c_str());
2969 [ + - ]: 11 : sps.push_back(NULL);
2970 : :
2971 [ + + + - ]: 11 : FTS *fts = fts_open ((char * const *)sps.data(),
2972 : : (traverse_logical ? FTS_LOGICAL : FTS_PHYSICAL|FTS_XDEV)
2973 : : | FTS_NOCHDIR /* multithreaded */,
2974 : : NULL);
2975 [ - + ]: 11 : if (fts == NULL)
2976 [ # # # # ]: 0 : throw libc_exception(errno, "cannot fts_open");
2977 : 11 : defer_dtor<FTS*,int> fts_cleanup (fts, fts_close);
2978 : :
2979 : : struct timespec ts_start, ts_end;
2980 : 11 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
2981 : 11 : unsigned fts_scanned = 0, fts_regex = 0;
2982 : :
2983 : : FTSENT *f;
2984 [ + - + + ]: 366 : while ((f = fts_read (fts)) != NULL)
2985 : : {
2986 [ - + ]: 355 : if (interrupted) break;
2987 : :
2988 [ - + ]: 355 : if (sigusr2 != forced_groom_count) // stop early if groom triggered
2989 : : {
2990 [ # # ]: 0 : scanq.clear(); // clear previously issued work for scanner threads
2991 : 0 : break;
2992 : : }
2993 : :
2994 : 355 : fts_scanned ++;
2995 : :
2996 [ + - ]: 355 : if (verbose > 2)
2997 [ + - + - : 355 : obatched(clog) << "fts traversing " << f->fts_path << endl;
+ - + - ]
2998 : :
2999 [ + - + + : 355 : switch (f->fts_info)
+ ]
3000 : : {
3001 : 173 : case FTS_F:
3002 : : {
3003 : : /* Found a file. Convert it to an absolute path, so
3004 : : the buildid database does not have relative path
3005 : : names that are unresolvable from a subsequent run
3006 : : in a different cwd. */
3007 : 173 : char *rp = realpath(f->fts_path, NULL);
3008 [ - + ]: 173 : if (rp == NULL)
3009 : 0 : continue; // ignore dangling symlink or such
3010 [ + - ]: 346 : string rps = string(rp);
3011 : 173 : free (rp);
3012 : :
3013 [ + - ]: 173 : bool ri = !regexec (&file_include_regex, rps.c_str(), 0, 0, 0);
3014 [ + - ]: 173 : bool rx = !regexec (&file_exclude_regex, rps.c_str(), 0, 0, 0);
3015 [ + - - + ]: 173 : if (!ri || rx)
3016 : : {
3017 [ # # ]: 0 : if (verbose > 3)
3018 [ # # # # ]: 0 : obatched(clog) << "fts skipped by regex "
3019 [ # # # # : 0 : << (!ri ? "I" : "") << (rx ? "X" : "") << endl;
# # # # #
# ]
3020 : 0 : fts_regex ++;
3021 [ # # ]: 0 : if (!ri)
3022 [ # # # # : 0 : inc_metric("traversed_total","type","file-skipped-I");
# # # # ]
3023 [ # # ]: 0 : if (rx)
3024 [ # # # # : 0 : inc_metric("traversed_total","type","file-skipped-X");
# # # # ]
3025 : : }
3026 : : else
3027 : : {
3028 [ + - + - ]: 173 : scanq.push_back (make_pair(rps, *f->fts_statp));
3029 [ + - + - : 173 : inc_metric("traversed_total","type","file");
+ - + - ]
3030 : : }
3031 : : }
3032 : 173 : break;
3033 : :
3034 : 0 : case FTS_ERR:
3035 : : case FTS_NS:
3036 : : // report on some types of errors because they may reflect fixable misconfiguration
3037 : : {
3038 [ # # # # : 0 : auto x = libc_exception(f->fts_errno, string("fts traversal ") + string(f->fts_path));
# # # # ]
3039 [ # # ]: 0 : x.report(cerr);
3040 : : }
3041 [ # # # # : 0 : inc_metric("traversed_total","type","error");
# # # # ]
3042 : 0 : break;
3043 : :
3044 : 18 : case FTS_SL: // ignore, but count because debuginfod -L would traverse these
3045 [ + - + - : 18 : inc_metric("traversed_total","type","symlink");
+ - + - ]
3046 : 18 : break;
3047 : :
3048 : 82 : case FTS_D: // ignore
3049 [ + - + - : 82 : inc_metric("traversed_total","type","directory");
+ - + - ]
3050 : 82 : break;
3051 : :
3052 : 82 : default: // ignore
3053 [ + - + - : 82 : inc_metric("traversed_total","type","other");
+ - + - ]
3054 : 82 : break;
3055 : : }
3056 : : }
3057 : 11 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
3058 : 11 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
3059 : :
3060 [ + - + - : 22 : obatched(clog) << "fts traversed source paths in " << deltas << "s, scanned=" << fts_scanned
+ - + - +
- ]
3061 [ + - + - : 11 : << ", regex-skipped=" << fts_regex << endl;
+ - ]
3062 : : }
3063 : :
3064 : :
3065 : : static void*
3066 : 3 : thread_main_fts_source_paths (void* arg)
3067 : : {
3068 : : (void) arg; // ignore; we operate on global data
3069 : :
3070 [ + - + - : 3 : set_metric("thread_tid", "role","traverse", tid());
+ - + - ]
3071 [ + - + - : 3 : add_metric("thread_count", "role", "traverse", 1);
+ - + - ]
3072 : :
3073 : 3 : time_t last_rescan = 0;
3074 : :
3075 [ + - ]: 35 : while (! interrupted)
3076 : : {
3077 : 35 : sleep (1);
3078 : 35 : scanq.wait_idle(); // don't start a new traversal while scanners haven't finished the job
3079 : 35 : scanq.done_idle(); // release the hounds
3080 [ + + ]: 35 : if (interrupted) break;
3081 : :
3082 : 32 : time_t now = time(NULL);
3083 : 32 : bool rescan_now = false;
3084 [ + + ]: 32 : if (last_rescan == 0) // at least one initial rescan is documented even for -t0
3085 : 3 : rescan_now = true;
3086 [ + + + + ]: 32 : if (rescan_s > 0 && (long)now > (long)(last_rescan + rescan_s))
3087 : 2 : rescan_now = true;
3088 [ + + ]: 32 : if (sigusr1 != forced_rescan_count)
3089 : : {
3090 : 9 : forced_rescan_count = sigusr1;
3091 : 9 : rescan_now = true;
3092 : : }
3093 [ + + ]: 32 : if (rescan_now)
3094 : : {
3095 [ + - + - : 12 : set_metric("thread_busy", "role","traverse", 1);
+ - + - ]
3096 : : try
3097 : : {
3098 [ + - ]: 12 : scan_source_paths();
3099 : : }
3100 [ - - ]: 0 : catch (const reportable_exception& e)
3101 : : {
3102 [ - - ]: 0 : e.report(cerr);
3103 : : }
3104 : 12 : last_rescan = time(NULL); // NB: now was before scanning
3105 : : // finished a traversal loop
3106 [ + - + - : 12 : inc_metric("thread_work_total", "role","traverse");
+ - + - ]
3107 [ + - + - : 12 : set_metric("thread_busy", "role","traverse", 0);
+ - + - ]
3108 : : }
3109 : : }
3110 : :
3111 : 3 : return 0;
3112 : : }
3113 : :
3114 : :
3115 : :
3116 : : ////////////////////////////////////////////////////////////////////////
3117 : :
3118 : : static void
3119 : 5 : database_stats_report()
3120 : : {
3121 : : sqlite_ps ps_query (db, "database-overview",
3122 [ + - + - : 20 : "select label,quantity from " BUILDIDS "_stats");
+ - ]
3123 : :
3124 [ + - + - : 5 : obatched(clog) << "database record counts:" << endl;
+ - ]
3125 : : while (1)
3126 : : {
3127 [ - + ]: 55 : if (interrupted) break;
3128 [ - + ]: 55 : if (sigusr1 != forced_rescan_count) // stop early if scan triggered
3129 : 0 : break;
3130 : :
3131 [ + - ]: 55 : int rc = ps_query.step();
3132 [ + + ]: 55 : if (rc == SQLITE_DONE) break;
3133 [ - + ]: 50 : if (rc != SQLITE_ROW)
3134 [ # # # # ]: 0 : throw sqlite_exception(rc, "step");
3135 : :
3136 [ + - ]: 100 : obatched(clog)
3137 [ + - + - ]: 100 : << right << setw(20) << ((const char*) sqlite3_column_text(ps_query, 0) ?: (const char*) "NULL")
3138 : : << " "
3139 [ + - + - : 100 : << (sqlite3_column_text(ps_query, 1) ?: (const unsigned char*) "NULL")
+ - + - +
- + - +
- ]
3140 [ + - ]: 50 : << endl;
3141 : :
3142 [ + - + - ]: 50 : set_metric("groom", "statistic",
3143 [ + - ]: 50 : ((const char*) sqlite3_column_text(ps_query, 0) ?: (const char*) "NULL"),
3144 [ + - + - : 150 : (sqlite3_column_double(ps_query, 1)));
+ - + - ]
3145 : 50 : }
3146 : 5 : }
3147 : :
3148 : :
3149 : : // Do a round of database grooming that might take many minutes to run.
3150 : 6 : void groom()
3151 : : {
3152 [ + - + - : 6 : obatched(clog) << "grooming database" << endl;
+ - ]
3153 : :
3154 : : struct timespec ts_start, ts_end;
3155 : 6 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
3156 : :
3157 : : // scan for files that have disappeared
3158 : : sqlite_ps files (db, "check old files", "select s.mtime, s.file, f.name from "
3159 : : BUILDIDS "_file_mtime_scanned s, " BUILDIDS "_files f "
3160 [ + - + - : 19 : "where f.id = s.file");
+ - ]
3161 [ + - + - : 19 : sqlite_ps files_del_f_de (db, "nuke f_de", "delete from " BUILDIDS "_f_de where file = ? and mtime = ?");
+ - ]
3162 [ + - + - : 19 : sqlite_ps files_del_r_de (db, "nuke r_de", "delete from " BUILDIDS "_r_de where file = ? and mtime = ?");
+ - ]
3163 : : sqlite_ps files_del_scan (db, "nuke f_m_s", "delete from " BUILDIDS "_file_mtime_scanned "
3164 [ + - + - : 19 : "where file = ? and mtime = ?");
+ - ]
3165 [ + - ]: 6 : files.reset();
3166 : : while(1)
3167 : : {
3168 [ - + ]: 154 : if (interrupted) break;
3169 : :
3170 [ + - ]: 154 : int rc = files.step();
3171 [ + + ]: 154 : if (rc != SQLITE_ROW)
3172 : 6 : break;
3173 : :
3174 [ + - ]: 148 : int64_t mtime = sqlite3_column_int64 (files, 0);
3175 [ + - ]: 148 : int64_t fileid = sqlite3_column_int64 (files, 1);
3176 [ + - + - ]: 148 : const char* filename = ((const char*) sqlite3_column_text (files, 2) ?: "");
3177 : : struct stat s;
3178 : 148 : rc = stat(filename, &s);
3179 [ + + - + ]: 148 : if (rc < 0 || (mtime != (int64_t) s.st_mtime))
3180 : : {
3181 [ + - ]: 16 : if (verbose > 2)
3182 [ + - + - : 16 : obatched(clog) << "groom: forgetting file=" << filename << " mtime=" << mtime << endl;
+ - + - +
- + - ]
3183 [ + - + - : 16 : files_del_f_de.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
3184 [ + - + - : 16 : files_del_r_de.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
3185 [ + - + - : 16 : files_del_scan.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
3186 [ + - + - : 16 : inc_metric("groomed_total", "decision", "stale");
+ - + - ]
3187 : : }
3188 : : else
3189 [ + - + - : 132 : inc_metric("groomed_total", "decision", "fresh");
+ - + - ]
3190 : :
3191 [ - + ]: 148 : if (sigusr1 != forced_rescan_count) // stop early if scan triggered
3192 : 0 : break;
3193 : 148 : }
3194 [ + - ]: 6 : files.reset();
3195 : :
3196 : : // delete buildids with no references in _r_de or _f_de tables;
3197 : : // cascades to _r_sref & _f_s records
3198 : : sqlite_ps buildids_del (db, "nuke orphan buildids",
3199 : : "delete from " BUILDIDS "_buildids "
3200 : : "where not exists (select 1 from " BUILDIDS "_f_de d where " BUILDIDS "_buildids.id = d.buildid) "
3201 [ + - + - : 19 : "and not exists (select 1 from " BUILDIDS "_r_de d where " BUILDIDS "_buildids.id = d.buildid)");
+ - ]
3202 [ + - + - ]: 6 : buildids_del.reset().step_ok_done();
3203 : :
3204 [ - + ]: 6 : if (interrupted) return;
3205 : :
3206 : : // NB: "vacuum" is too heavy for even daily runs: it rewrites the entire db, so is done as maxigroom -G
3207 [ + - + - : 24 : sqlite_ps g1 (db, "incremental vacuum", "pragma incremental_vacuum");
+ - ]
3208 [ + - + - ]: 6 : g1.reset().step_ok_done();
3209 [ + - + - : 24 : sqlite_ps g2 (db, "optimize", "pragma optimize");
+ - ]
3210 [ + - + + ]: 6 : g2.reset().step_ok_done();
3211 [ + - + - : 15 : sqlite_ps g3 (db, "wal checkpoint", "pragma wal_checkpoint=truncate");
+ - ]
3212 [ + - + - ]: 5 : g3.reset().step_ok_done();
3213 : :
3214 [ + - ]: 5 : database_stats_report();
3215 : :
3216 [ + - + - ]: 5 : (void) statfs_free_enough_p(db_path, "database"); // report sqlite filesystem size
3217 : :
3218 [ + - ]: 5 : sqlite3_db_release_memory(db); // shrink the process if possible
3219 [ + - ]: 5 : sqlite3_db_release_memory(dbq); // ... for both connections
3220 [ + - ]: 5 : debuginfod_pool_groom(); // and release any debuginfod_client objects we've been holding onto
3221 : :
3222 [ + - ]: 5 : fdcache.limit(0,0); // release the fdcache contents
3223 [ + - ]: 5 : fdcache.limit(fdcache_fds,fdcache_mbs); // restore status quo parameters
3224 : :
3225 : 5 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
3226 : 5 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
3227 : :
3228 [ + - + - : 5 : obatched(clog) << "groomed database in " << deltas << "s" << endl;
+ - + - +
- ]
3229 : : }
3230 : :
3231 : :
3232 : : static void*
3233 : 3 : thread_main_groom (void* /*arg*/)
3234 : : {
3235 [ + - + - : 3 : set_metric("thread_tid", "role", "groom", tid());
+ - + - ]
3236 [ + - + - : 3 : add_metric("thread_count", "role", "groom", 1);
+ - + - ]
3237 : :
3238 : 3 : time_t last_groom = 0;
3239 : :
3240 : : while (1)
3241 : : {
3242 : 35 : sleep (1);
3243 : 35 : scanq.wait_idle(); // PR25394: block scanners during grooming!
3244 [ + + ]: 35 : if (interrupted) break;
3245 : :
3246 : 32 : time_t now = time(NULL);
3247 : 32 : bool groom_now = false;
3248 [ + + ]: 32 : if (last_groom == 0) // at least one initial groom is documented even for -g0
3249 : 3 : groom_now = true;
3250 [ + + + + ]: 32 : if (groom_s > 0 && (long)now > (long)(last_groom + groom_s))
3251 : 2 : groom_now = true;
3252 [ + + ]: 32 : if (sigusr2 != forced_groom_count)
3253 : : {
3254 : 3 : forced_groom_count = sigusr2;
3255 : 3 : groom_now = true;
3256 : : }
3257 [ + + ]: 32 : if (groom_now)
3258 : : {
3259 [ + - + - : 6 : set_metric("thread_busy", "role", "groom", 1);
+ - + - ]
3260 : : try
3261 : : {
3262 [ + + ]: 6 : groom ();
3263 : : }
3264 [ - + ]: 1 : catch (const sqlite_exception& e)
3265 : : {
3266 [ + - + - : 1 : obatched(cerr) << e.message << endl;
+ - ]
3267 : : }
3268 : 6 : last_groom = time(NULL); // NB: now was before grooming
3269 : : // finished a grooming loop
3270 [ + - + - : 6 : inc_metric("thread_work_total", "role", "groom");
+ - + - ]
3271 [ + - + - : 6 : set_metric("thread_busy", "role", "groom", 0);
+ - + - ]
3272 : : }
3273 : :
3274 : 32 : scanq.done_idle();
3275 : 32 : }
3276 : :
3277 : 3 : return 0;
3278 : : }
3279 : :
3280 : :
3281 : : ////////////////////////////////////////////////////////////////////////
3282 : :
3283 : :
3284 : : static void
3285 : 3 : signal_handler (int /* sig */)
3286 : : {
3287 : 3 : interrupted ++;
3288 : :
3289 [ + - ]: 3 : if (db)
3290 : 3 : sqlite3_interrupt (db);
3291 [ + - ]: 3 : if (dbq)
3292 : 3 : sqlite3_interrupt (dbq);
3293 : :
3294 : : // NB: don't do anything else in here
3295 : 3 : }
3296 : :
3297 : : static void
3298 : 9 : sigusr1_handler (int /* sig */)
3299 : : {
3300 : 9 : sigusr1 ++;
3301 : : // NB: don't do anything else in here
3302 : 9 : }
3303 : :
3304 : : static void
3305 : 3 : sigusr2_handler (int /* sig */)
3306 : : {
3307 : 3 : sigusr2 ++;
3308 : : // NB: don't do anything else in here
3309 : 3 : }
3310 : :
3311 : :
3312 : :
3313 : :
3314 : :
3315 : : // A user-defined sqlite function, to score the sharedness of the
3316 : : // prefix of two strings. This is used to compare candidate debuginfo
3317 : : // / source-rpm names, so that the closest match
3318 : : // (directory-topology-wise closest) is found. This is important in
3319 : : // case the same sref (source file name) is in many -debuginfo or
3320 : : // -debugsource RPMs, such as when multiple versions/releases of the
3321 : : // same package are in the database.
3322 : :
3323 : 109 : static void sqlite3_sharedprefix_fn (sqlite3_context* c, int argc, sqlite3_value** argv)
3324 : : {
3325 [ - + ]: 109 : if (argc != 2)
3326 : 0 : sqlite3_result_error(c, "expect 2 string arguments", -1);
3327 [ + - + + ]: 218 : else if ((sqlite3_value_type(argv[0]) != SQLITE_TEXT) ||
3328 [ + + ]: 109 : (sqlite3_value_type(argv[1]) != SQLITE_TEXT))
3329 : 3 : sqlite3_result_null(c);
3330 : : else
3331 : : {
3332 : 106 : const unsigned char* a = sqlite3_value_text (argv[0]);
3333 : 106 : const unsigned char* b = sqlite3_value_text (argv[1]);
3334 : 106 : int i = 0;
3335 [ + + ]: 8516 : while (*a++ == *b++)
3336 : 8410 : i++;
3337 : 106 : sqlite3_result_int (c, i);
3338 : : }
3339 : 109 : }
3340 : :
3341 : :
3342 : : int
3343 : 3 : main (int argc, char *argv[])
3344 : : {
3345 : 3 : (void) setlocale (LC_ALL, "");
3346 : 3 : (void) bindtextdomain (PACKAGE_TARNAME, LOCALEDIR);
3347 : 3 : (void) textdomain (PACKAGE_TARNAME);
3348 : :
3349 : : /* Tell the library which version we are expecting. */
3350 [ + - ]: 3 : elf_version (EV_CURRENT);
3351 : :
3352 [ - + + - ]: 3 : tmpdir = string(getenv("TMPDIR") ?: "/tmp");
3353 : :
3354 : : /* Set computed default values. */
3355 [ + - + - : 3 : db_path = string(getenv("HOME") ?: "/") + string("/.debuginfod.sqlite"); /* XDG? */
+ - + - ]
3356 [ + - ]: 3 : int rc = regcomp (& file_include_regex, ".*", REG_EXTENDED|REG_NOSUB); // match everything
3357 [ - + ]: 3 : if (rc != 0)
3358 : : error (EXIT_FAILURE, 0, "regcomp failure: %d", rc);
3359 [ + - ]: 3 : rc = regcomp (& file_exclude_regex, "^$", REG_EXTENDED|REG_NOSUB); // match nothing
3360 [ - + ]: 3 : if (rc != 0)
3361 : : error (EXIT_FAILURE, 0, "regcomp failure: %d", rc);
3362 : :
3363 : : // default parameters for fdcache are computed from system stats
3364 : : struct statfs sfs;
3365 : 3 : rc = statfs(tmpdir.c_str(), &sfs);
3366 [ - + ]: 3 : if (rc < 0)
3367 : 0 : fdcache_mbs = 1024; // 1 gigabyte
3368 : : else
3369 : 3 : fdcache_mbs = sfs.f_bavail * sfs.f_bsize / 1024 / 1024 / 4; // 25% of free space
3370 : 3 : fdcache_mintmp = 25; // emergency flush at 25% remaining (75% full)
3371 : 3 : fdcache_prefetch = 64; // guesstimate storage is this much less costly than re-decompression
3372 : 3 : fdcache_fds = (concurrency + fdcache_prefetch) * 2;
3373 : :
3374 : : /* Parse and process arguments. */
3375 : : int remaining;
3376 : 3 : argp_program_version_hook = print_version; // this works
3377 [ + - ]: 3 : (void) argp_parse (&argp, argc, argv, ARGP_IN_ORDER, &remaining, NULL);
3378 [ - + ]: 3 : if (remaining != argc)
3379 : 0 : error (EXIT_FAILURE, 0,
3380 : 0 : "unexpected argument: %s", argv[remaining]);
3381 : :
3382 [ - + - - : 3 : if (scan_archives.size()==0 && !scan_files && source_paths.size()>0)
- - - + ]
3383 [ # # # # : 0 : obatched(clog) << "warning: without -F -R -U -Z, ignoring PATHs" << endl;
# # ]
3384 : :
3385 [ + - ]: 3 : fdcache.limit(fdcache_fds, fdcache_mbs);
3386 : :
3387 : 3 : (void) signal (SIGPIPE, SIG_IGN); // microhttpd can generate it incidentally, ignore
3388 : 3 : (void) signal (SIGINT, signal_handler); // ^C
3389 : 3 : (void) signal (SIGHUP, signal_handler); // EOF
3390 : 3 : (void) signal (SIGTERM, signal_handler); // systemd
3391 : 3 : (void) signal (SIGUSR1, sigusr1_handler); // end-user
3392 : 3 : (void) signal (SIGUSR2, sigusr2_handler); // end-user
3393 : :
3394 : : /* Get database ready. */
3395 [ + - ]: 3 : rc = sqlite3_open_v2 (db_path.c_str(), &db, (SQLITE_OPEN_READWRITE
3396 : : |SQLITE_OPEN_URI
3397 : : |SQLITE_OPEN_PRIVATECACHE
3398 : : |SQLITE_OPEN_CREATE
3399 : : |SQLITE_OPEN_FULLMUTEX), /* thread-safe */
3400 : : NULL);
3401 [ - + ]: 3 : if (rc == SQLITE_CORRUPT)
3402 : : {
3403 : 0 : (void) unlink (db_path.c_str());
3404 [ # # ]: 0 : error (EXIT_FAILURE, 0,
3405 : : "cannot open %s, deleted database: %s", db_path.c_str(), sqlite3_errmsg(db));
3406 : : }
3407 [ - + ]: 3 : else if (rc)
3408 : : {
3409 [ # # ]: 0 : error (EXIT_FAILURE, 0,
3410 : : "cannot open %s, consider deleting database: %s", db_path.c_str(), sqlite3_errmsg(db));
3411 : : }
3412 : :
3413 : : // open the readonly query variant
3414 : : // NB: PRIVATECACHE allows web queries to operate in parallel with
3415 : : // much other grooming/scanning operation.
3416 [ + - ]: 3 : rc = sqlite3_open_v2 (db_path.c_str(), &dbq, (SQLITE_OPEN_READONLY
3417 : : |SQLITE_OPEN_URI
3418 : : |SQLITE_OPEN_PRIVATECACHE
3419 : : |SQLITE_OPEN_FULLMUTEX), /* thread-safe */
3420 : : NULL);
3421 [ - + ]: 3 : if (rc)
3422 : : {
3423 [ # # ]: 0 : error (EXIT_FAILURE, 0,
3424 : : "cannot open %s, consider deleting database: %s", db_path.c_str(), sqlite3_errmsg(dbq));
3425 : : }
3426 : :
3427 : :
3428 [ + - + - : 3 : obatched(clog) << "opened database " << db_path << endl;
+ - + - ]
3429 [ + - + - : 3 : obatched(clog) << "sqlite version " << sqlite3_version << endl;
+ - + - ]
3430 : :
3431 : : // add special string-prefix-similarity function used in rpm sref/sdef resolution
3432 [ + - ]: 3 : rc = sqlite3_create_function(dbq, "sharedprefix", 2, SQLITE_UTF8, NULL,
3433 : : & sqlite3_sharedprefix_fn, NULL, NULL);
3434 [ - + ]: 3 : if (rc != SQLITE_OK)
3435 [ # # ]: 0 : error (EXIT_FAILURE, 0,
3436 : : "cannot create sharedprefix function: %s", sqlite3_errmsg(dbq));
3437 : :
3438 [ + + ]: 3 : if (verbose > 3)
3439 [ + - + - : 1 : obatched(clog) << "ddl: " << DEBUGINFOD_SQLITE_DDL << endl;
+ - + - ]
3440 [ + - ]: 3 : rc = sqlite3_exec (db, DEBUGINFOD_SQLITE_DDL, NULL, NULL, NULL);
3441 [ - + ]: 3 : if (rc != SQLITE_OK)
3442 : : {
3443 [ # # ]: 0 : error (EXIT_FAILURE, 0,
3444 : : "cannot run database schema ddl: %s", sqlite3_errmsg(db));
3445 : : }
3446 : :
3447 : : // Start httpd server threads. Separate pool for IPv4 and IPv6, in
3448 : : // case the host only has one protocol stack.
3449 [ + - ]: 3 : MHD_Daemon *d4 = MHD_start_daemon (MHD_USE_THREAD_PER_CONNECTION
3450 : : #if MHD_VERSION >= 0x00095300
3451 : : | MHD_USE_INTERNAL_POLLING_THREAD
3452 : : #else
3453 : : | MHD_USE_SELECT_INTERNALLY
3454 : : #endif
3455 : : | MHD_USE_DEBUG, /* report errors to stderr */
3456 : : http_port,
3457 : : NULL, NULL, /* default accept policy */
3458 : : handler_cb, NULL, /* handler callback */
3459 : : MHD_OPTION_END);
3460 [ + - ]: 3 : MHD_Daemon *d6 = MHD_start_daemon (MHD_USE_THREAD_PER_CONNECTION
3461 : : #if MHD_VERSION >= 0x00095300
3462 : : | MHD_USE_INTERNAL_POLLING_THREAD
3463 : : #else
3464 : : | MHD_USE_SELECT_INTERNALLY
3465 : : #endif
3466 : : | MHD_USE_IPv6
3467 : : | MHD_USE_DEBUG, /* report errors to stderr */
3468 : : http_port,
3469 : : NULL, NULL, /* default accept policy */
3470 : : handler_cb, NULL, /* handler callback */
3471 : : MHD_OPTION_END);
3472 : :
3473 [ - + - - ]: 3 : if (d4 == NULL && d6 == NULL) // neither ipv4 nor ipv6? boo
3474 : : {
3475 : 0 : sqlite3 *database = db;
3476 : 0 : sqlite3 *databaseq = dbq;
3477 : 0 : db = dbq = 0; // for signal_handler not to freak
3478 [ # # ]: 0 : sqlite3_close (databaseq);
3479 [ # # ]: 0 : sqlite3_close (database);
3480 : 0 : error (EXIT_FAILURE, 0, "cannot start http server at port %d", http_port);
3481 : : }
3482 : :
3483 [ + - + - ]: 6 : obatched(clog) << "started http server on "
3484 : : << (d4 != NULL ? "IPv4 " : "")
3485 : : << (d6 != NULL ? "IPv6 " : "")
3486 [ + - + - : 3 : << "port=" << http_port << endl;
+ - + - +
- + - +
- ]
3487 : :
3488 : : // add maxigroom sql if -G given
3489 [ - + ]: 3 : if (maxigroom)
3490 : : {
3491 [ # # # # : 0 : obatched(clog) << "maxigrooming database, please wait." << endl;
# # ]
3492 [ # # # # ]: 0 : extra_ddl.push_back("create index if not exists " BUILDIDS "_r_sref_arc on " BUILDIDS "_r_sref(artifactsrc);");
3493 [ # # # # ]: 0 : extra_ddl.push_back("delete from " BUILDIDS "_r_sdef where not exists (select 1 from " BUILDIDS "_r_sref b where " BUILDIDS "_r_sdef.content = b.artifactsrc);");
3494 [ # # # # ]: 0 : extra_ddl.push_back("drop index if exists " BUILDIDS "_r_sref_arc;");
3495 : :
3496 : : // NB: we don't maxigroom the _files interning table. It'd require a temp index on all the
3497 : : // tables that have file foreign-keys, which is a lot.
3498 : :
3499 : : // NB: with =delete, may take up 3x disk space total during vacuum process
3500 : : // vs. =off (only 2x but may corrupt database if program dies mid-vacuum)
3501 : : // vs. =wal (>3x observed, but safe)
3502 [ # # # # ]: 0 : extra_ddl.push_back("pragma journal_mode=delete;");
3503 [ # # # # ]: 0 : extra_ddl.push_back("vacuum;");
3504 [ # # # # ]: 0 : extra_ddl.push_back("pragma journal_mode=wal;");
3505 : : }
3506 : :
3507 : : // run extra -D sql if given
3508 [ - + ]: 3 : for (auto&& i: extra_ddl)
3509 : : {
3510 [ # # ]: 0 : if (verbose > 1)
3511 [ # # # # : 0 : obatched(clog) << "extra ddl:\n" << i << endl;
# # # # ]
3512 [ # # ]: 0 : rc = sqlite3_exec (db, i.c_str(), NULL, NULL, NULL);
3513 [ # # # # : 0 : if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)
# # ]
3514 [ # # ]: 0 : error (0, 0,
3515 : : "warning: cannot run database extra ddl %s: %s", i.c_str(), sqlite3_errmsg(db));
3516 : : }
3517 : :
3518 [ - + ]: 3 : if (maxigroom)
3519 [ # # # # : 0 : obatched(clog) << "maxigroomed database" << endl;
# # ]
3520 : :
3521 [ + - + - : 3 : obatched(clog) << "search concurrency " << concurrency << endl;
+ - + - ]
3522 [ + - + - : 3 : obatched(clog) << "rescan time " << rescan_s << endl;
+ - + - ]
3523 [ + - + - : 3 : obatched(clog) << "fdcache fds " << fdcache_fds << endl;
+ - + - ]
3524 [ + - + - : 3 : obatched(clog) << "fdcache mbs " << fdcache_mbs << endl;
+ - + - ]
3525 [ + - + - : 3 : obatched(clog) << "fdcache prefetch " << fdcache_prefetch << endl;
+ - + - ]
3526 [ + - + - : 3 : obatched(clog) << "fdcache tmpdir " << tmpdir << endl;
+ - + - ]
3527 [ + - + - : 3 : obatched(clog) << "fdcache tmpdir min% " << fdcache_mintmp << endl;
+ - + - ]
3528 [ + - + - : 3 : obatched(clog) << "groom time " << groom_s << endl;
+ - + - ]
3529 [ + - ]: 3 : if (scan_archives.size()>0)
3530 : : {
3531 [ + - ]: 6 : obatched ob(clog);
3532 [ + - ]: 3 : auto& o = ob << "scanning archive types ";
3533 [ + + ]: 12 : for (auto&& arch : scan_archives)
3534 [ + - + - : 9 : o << arch.first << "(" << arch.second << ") ";
+ - + - ]
3535 [ + - ]: 3 : o << endl;
3536 : : }
3537 : 3 : const char* du = getenv(DEBUGINFOD_URLS_ENV_VAR);
3538 [ + - + + ]: 3 : if (du && du[0] != '\0') // set to non-empty string?
3539 [ + - + - : 2 : obatched(clog) << "upstream debuginfod servers: " << du << endl;
+ - + - ]
3540 : :
3541 : 3 : vector<pthread_t> all_threads;
3542 : :
3543 : : pthread_t pt;
3544 : 3 : rc = pthread_create (& pt, NULL, thread_main_groom, NULL);
3545 [ - + ]: 3 : if (rc)
3546 : : error (EXIT_FAILURE, rc, "cannot spawn thread to groom database\n");
3547 : : else
3548 : : {
3549 : : #ifdef HAVE_PTHREAD_SETNAME_NP
3550 : 3 : (void) pthread_setname_np (pt, "groom");
3551 : : #endif
3552 [ + - ]: 3 : all_threads.push_back(pt);
3553 : : }
3554 : :
3555 [ - + - - : 3 : if (scan_files || scan_archives.size() > 0)
+ - ]
3556 : : {
3557 : 3 : rc = pthread_create (& pt, NULL, thread_main_fts_source_paths, NULL);
3558 [ - + ]: 3 : if (rc)
3559 : : error (EXIT_FAILURE, rc, "cannot spawn thread to traverse source paths\n");
3560 : : #ifdef HAVE_PTHREAD_SETNAME_NP
3561 : 3 : (void) pthread_setname_np (pt, "traverse");
3562 : : #endif
3563 [ + - ]: 3 : all_threads.push_back(pt);
3564 : :
3565 [ + + ]: 15 : for (unsigned i=0; i<concurrency; i++)
3566 : : {
3567 : 12 : rc = pthread_create (& pt, NULL, thread_main_scanner, NULL);
3568 [ - + ]: 12 : if (rc)
3569 : : error (EXIT_FAILURE, rc, "cannot spawn thread to scan source files / archives\n");
3570 : : #ifdef HAVE_PTHREAD_SETNAME_NP
3571 : 12 : (void) pthread_setname_np (pt, "scan");
3572 : : #endif
3573 [ + - ]: 12 : all_threads.push_back(pt);
3574 : : }
3575 : : }
3576 : :
3577 : : /* Trivial main loop! */
3578 [ + - + - ]: 3 : set_metric("ready", 1);
3579 [ + + ]: 18 : while (! interrupted)
3580 [ + - ]: 15 : pause ();
3581 [ + - ]: 3 : scanq.nuke(); // wake up any remaining scanq-related threads, let them die
3582 [ + - + - ]: 3 : set_metric("ready", 0);
3583 : :
3584 [ + - ]: 3 : if (verbose)
3585 [ + - + - : 3 : obatched(clog) << "stopping" << endl;
+ - ]
3586 : :
3587 : : /* Join all our threads. */
3588 [ + + ]: 21 : for (auto&& it : all_threads)
3589 [ + - ]: 18 : pthread_join (it, NULL);
3590 : :
3591 : : /* Stop all the web service threads. */
3592 [ + - + - ]: 3 : if (d4) MHD_stop_daemon (d4);
3593 [ + - + - ]: 3 : if (d6) MHD_stop_daemon (d6);
3594 : :
3595 : : /* With all threads known dead, we can clean up the global resources. */
3596 [ + - ]: 3 : rc = sqlite3_exec (db, DEBUGINFOD_SQLITE_CLEANUP_DDL, NULL, NULL, NULL);
3597 [ - + ]: 3 : if (rc != SQLITE_OK)
3598 : : {
3599 [ # # ]: 0 : error (0, 0,
3600 : : "warning: cannot run database cleanup ddl: %s", sqlite3_errmsg(db));
3601 : : }
3602 : :
3603 : : // NB: no problem with unconditional free here - an earlier failed regcomp would exit program
3604 [ + - ]: 3 : (void) regfree (& file_include_regex);
3605 [ + - ]: 3 : (void) regfree (& file_exclude_regex);
3606 : :
3607 : 3 : sqlite3 *database = db;
3608 : 3 : sqlite3 *databaseq = dbq;
3609 : 3 : db = dbq = 0; // for signal_handler not to freak
3610 [ + - ]: 3 : (void) sqlite3_close (databaseq);
3611 [ + - ]: 3 : (void) sqlite3_close (database);
3612 : :
3613 : 3 : return 0;
3614 : : }
|
