Branch data Line data Source code
1 : : /* Debuginfo-over-http server.
2 : : Copyright (C) 2019-2021 Red Hat, Inc.
3 : : Copyright (C) 2021 Mark J. Wielaard <mark@klomp.org>
4 : : This file is part of elfutils.
5 : :
6 : : This file is free software; you can redistribute it and/or modify
7 : : it under the terms of the GNU General Public License as published by
8 : : the Free Software Foundation; either version 3 of the License, or
9 : : (at your option) any later version.
10 : :
11 : : elfutils is distributed in the hope that it will be useful, but
12 : : WITHOUT ANY WARRANTY; without even the implied warranty of
13 : : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : : GNU General Public License for more details.
15 : :
16 : : You should have received a copy of the GNU General Public License
17 : : along with this program. If not, see <http://www.gnu.org/licenses/>. */
18 : :
19 : :
20 : : /* cargo-cult from libdwfl linux-kernel-modules.c */
21 : : /* In case we have a bad fts we include this before config.h because it
22 : : can't handle _FILE_OFFSET_BITS.
23 : : Everything we need here is fine if its declarations just come first.
24 : : Also, include sys/types.h before fts. On some systems fts.h is not self
25 : : contained. */
26 : : #ifdef BAD_FTS
27 : : #include <sys/types.h>
28 : : #include <fts.h>
29 : : #endif
30 : :
31 : : #ifdef HAVE_CONFIG_H
32 : : #include "config.h"
33 : : #endif
34 : :
35 : : extern "C" {
36 : : #include "printversion.h"
37 : : #include "system.h"
38 : : }
39 : :
40 : : #include "debuginfod.h"
41 : : #include <dwarf.h>
42 : :
43 : : #include <argp.h>
44 : : #ifdef __GNUC__
45 : : #undef __attribute__ /* glibc bug - rhbz 1763325 */
46 : : #endif
47 : :
48 : : #include <unistd.h>
49 : : #include <stdlib.h>
50 : : #include <libintl.h>
51 : : #include <locale.h>
52 : : #include <pthread.h>
53 : : #include <signal.h>
54 : : #include <sys/stat.h>
55 : : #include <sys/time.h>
56 : : #include <sys/vfs.h>
57 : : #include <unistd.h>
58 : : #include <fcntl.h>
59 : : #include <netdb.h>
60 : :
61 : :
62 : : /* If fts.h is included before config.h, its indirect inclusions may not
63 : : give us the right LFS aliases of these functions, so map them manually. */
64 : : #ifdef BAD_FTS
65 : : #ifdef _FILE_OFFSET_BITS
66 : : #define open open64
67 : : #define fopen fopen64
68 : : #endif
69 : : #else
70 : : #include <sys/types.h>
71 : : #include <fts.h>
72 : : #endif
73 : :
74 : : #include <cstring>
75 : : #include <vector>
76 : : #include <set>
77 : : #include <map>
78 : : #include <string>
79 : : #include <iostream>
80 : : #include <iomanip>
81 : : #include <ostream>
82 : : #include <sstream>
83 : : #include <mutex>
84 : : #include <deque>
85 : : #include <condition_variable>
86 : : #include <thread>
87 : : // #include <regex> // on rhel7 gcc 4.8, not competent
88 : : #include <regex.h>
89 : : // #include <algorithm>
90 : : using namespace std;
91 : :
92 : : #include <gelf.h>
93 : : #include <libdwelf.h>
94 : :
95 : : #include <microhttpd.h>
96 : :
97 : : #if MHD_VERSION >= 0x00097002
98 : : // libmicrohttpd 0.9.71 broke API
99 : : #define MHD_RESULT enum MHD_Result
100 : : #else
101 : : #define MHD_RESULT int
102 : : #endif
103 : :
104 : : #include <curl/curl.h>
105 : : #include <archive.h>
106 : : #include <archive_entry.h>
107 : : #include <sqlite3.h>
108 : :
109 : : #ifdef __linux__
110 : : #include <sys/syscall.h>
111 : : #endif
112 : :
113 : : #ifdef __linux__
114 : : #define tid() syscall(SYS_gettid)
115 : : #else
116 : : #define tid() pthread_self()
117 : : #endif
118 : :
119 : :
120 : : inline bool
121 : 1308 : string_endswith(const string& haystack, const string& needle)
122 : : {
123 [ + # + + ]: 1308 : return (haystack.size() >= needle.size() &&
124 [ + + ]: 1312 : equal(haystack.end()-needle.size(), haystack.end(),
125 : 1308 : needle.begin()));
126 : : }
127 : :
128 : :
129 : : // Roll this identifier for every sqlite schema incompatibility.
130 : : #define BUILDIDS "buildids9"
131 : :
132 : : #if SQLITE_VERSION_NUMBER >= 3008000
133 : : #define WITHOUT_ROWID "without rowid"
134 : : #else
135 : : #define WITHOUT_ROWID ""
136 : : #endif
137 : :
138 : : static const char DEBUGINFOD_SQLITE_DDL[] =
139 : : "pragma foreign_keys = on;\n"
140 : : "pragma synchronous = 0;\n" // disable fsync()s - this cache is disposable across a machine crash
141 : : "pragma journal_mode = wal;\n" // https://sqlite.org/wal.html
142 : : "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file
143 : : "pragma journal_size_limit = 0;\n" // limit steady state file (between grooming, which also =truncate's)
144 : : "pragma auto_vacuum = incremental;\n" // https://sqlite.org/pragma.html
145 : : "pragma busy_timeout = 1000;\n" // https://sqlite.org/pragma.html
146 : : // NB: all these are overridable with -D option
147 : :
148 : : // Normalization table for interning file names
149 : : "create table if not exists " BUILDIDS "_files (\n"
150 : : " id integer primary key not null,\n"
151 : : " name text unique not null\n"
152 : : " );\n"
153 : : // Normalization table for interning buildids
154 : : "create table if not exists " BUILDIDS "_buildids (\n"
155 : : " id integer primary key not null,\n"
156 : : " hex text unique not null);\n"
157 : : // Track the completion of scanning of a given file & sourcetype at given time
158 : : "create table if not exists " BUILDIDS "_file_mtime_scanned (\n"
159 : : " mtime integer not null,\n"
160 : : " file integer not null,\n"
161 : : " size integer not null,\n" // in bytes
162 : : " sourcetype text(1) not null\n"
163 : : " check (sourcetype IN ('F', 'R')),\n"
164 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
165 : : " primary key (file, mtime, sourcetype)\n"
166 : : " ) " WITHOUT_ROWID ";\n"
167 : : "create table if not exists " BUILDIDS "_f_de (\n"
168 : : " buildid integer not null,\n"
169 : : " debuginfo_p integer not null,\n"
170 : : " executable_p integer not null,\n"
171 : : " file integer not null,\n"
172 : : " mtime integer not null,\n"
173 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
174 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
175 : : " primary key (buildid, file, mtime)\n"
176 : : " ) " WITHOUT_ROWID ";\n"
177 : : "create table if not exists " BUILDIDS "_f_s (\n"
178 : : " buildid integer not null,\n"
179 : : " artifactsrc integer not null,\n"
180 : : " file integer not null,\n" // NB: not necessarily entered into _mtime_scanned
181 : : " mtime integer not null,\n"
182 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
183 : : " foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
184 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
185 : : " primary key (buildid, artifactsrc, file, mtime)\n"
186 : : " ) " WITHOUT_ROWID ";\n"
187 : : "create table if not exists " BUILDIDS "_r_de (\n"
188 : : " buildid integer not null,\n"
189 : : " debuginfo_p integer not null,\n"
190 : : " executable_p integer not null,\n"
191 : : " file integer not null,\n"
192 : : " mtime integer not null,\n"
193 : : " content integer not null,\n"
194 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
195 : : " foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
196 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
197 : : " primary key (buildid, debuginfo_p, executable_p, file, content, mtime)\n"
198 : : " ) " WITHOUT_ROWID ";\n"
199 : : "create table if not exists " BUILDIDS "_r_sref (\n" // outgoing dwarf sourcefile references from rpm
200 : : " buildid integer not null,\n"
201 : : " artifactsrc integer not null,\n"
202 : : " foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
203 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
204 : : " primary key (buildid, artifactsrc)\n"
205 : : " ) " WITHOUT_ROWID ";\n"
206 : : "create table if not exists " BUILDIDS "_r_sdef (\n" // rpm contents that may satisfy sref
207 : : " file integer not null,\n"
208 : : " mtime integer not null,\n"
209 : : " content integer not null,\n"
210 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
211 : : " foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
212 : : " primary key (content, file, mtime)\n"
213 : : " ) " WITHOUT_ROWID ";\n"
214 : : // create views to glue together some of the above tables, for webapi D queries
215 : : "create view if not exists " BUILDIDS "_query_d as \n"
216 : : "select\n"
217 : : " b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"
218 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_f_de n\n"
219 : : " where b.id = n.buildid and f0.id = n.file and n.debuginfo_p = 1\n"
220 : : "union all select\n"
221 : : " b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"
222 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_r_de n\n"
223 : : " where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.debuginfo_p = 1\n"
224 : : ";"
225 : : // ... and for E queries
226 : : "create view if not exists " BUILDIDS "_query_e as \n"
227 : : "select\n"
228 : : " b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"
229 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_f_de n\n"
230 : : " where b.id = n.buildid and f0.id = n.file and n.executable_p = 1\n"
231 : : "union all select\n"
232 : : " b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"
233 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_r_de n\n"
234 : : " where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.executable_p = 1\n"
235 : : ";"
236 : : // ... and for S queries
237 : : "create view if not exists " BUILDIDS "_query_s as \n"
238 : : "select\n"
239 : : " b.hex as buildid, fs.name as artifactsrc, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1, null as source0ref\n"
240 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files fs, " BUILDIDS "_f_s n\n"
241 : : " where b.id = n.buildid and f0.id = n.file and fs.id = n.artifactsrc\n"
242 : : "union all select\n"
243 : : " b.hex as buildid, f1.name as artifactsrc, 'R' as sourcetype, f0.name as source0, sd.mtime as mtime, f1.name as source1, fsref.name as source0ref\n"
244 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_files fsref, "
245 : : " " BUILDIDS "_r_sdef sd, " BUILDIDS "_r_sref sr, " BUILDIDS "_r_de sde\n"
246 : : " where b.id = sr.buildid and f0.id = sd.file and fsref.id = sde.file and f1.id = sd.content\n"
247 : : " and sr.artifactsrc = sd.content and sde.buildid = sr.buildid\n"
248 : : ";"
249 : : // and for startup overview counts
250 : : "drop view if exists " BUILDIDS "_stats;\n"
251 : : "create view if not exists " BUILDIDS "_stats as\n"
252 : : " select 'file d/e' as label,count(*) as quantity from " BUILDIDS "_f_de\n"
253 : : "union all select 'file s',count(*) from " BUILDIDS "_f_s\n"
254 : : "union all select 'archive d/e',count(*) from " BUILDIDS "_r_de\n"
255 : : "union all select 'archive sref',count(*) from " BUILDIDS "_r_sref\n"
256 : : "union all select 'archive sdef',count(*) from " BUILDIDS "_r_sdef\n"
257 : : "union all select 'buildids',count(*) from " BUILDIDS "_buildids\n"
258 : : "union all select 'filenames',count(*) from " BUILDIDS "_files\n"
259 : : "union all select 'files scanned (#)',count(*) from " BUILDIDS "_file_mtime_scanned\n"
260 : : "union all select 'files scanned (mb)',coalesce(sum(size)/1024/1024,0) from " BUILDIDS "_file_mtime_scanned\n"
261 : : #if SQLITE_VERSION_NUMBER >= 3016000
262 : : "union all select 'index db size (mb)',page_count*page_size/1024/1024 as size FROM pragma_page_count(), pragma_page_size()\n"
263 : : #endif
264 : : ";\n"
265 : :
266 : : // schema change history & garbage collection
267 : : //
268 : : // XXX: we could have migration queries here to bring prior-schema
269 : : // data over instead of just dropping it.
270 : : //
271 : : // buildids9: widen the mtime_scanned table
272 : : "" // <<< we are here
273 : : // buildids8: slim the sref table
274 : : "drop table if exists buildids8_f_de;\n"
275 : : "drop table if exists buildids8_f_s;\n"
276 : : "drop table if exists buildids8_r_de;\n"
277 : : "drop table if exists buildids8_r_sref;\n"
278 : : "drop table if exists buildids8_r_sdef;\n"
279 : : "drop table if exists buildids8_file_mtime_scanned;\n"
280 : : "drop table if exists buildids8_files;\n"
281 : : "drop table if exists buildids8_buildids;\n"
282 : : // buildids7: separate _norm table into dense subtype tables
283 : : "drop table if exists buildids7_f_de;\n"
284 : : "drop table if exists buildids7_f_s;\n"
285 : : "drop table if exists buildids7_r_de;\n"
286 : : "drop table if exists buildids7_r_sref;\n"
287 : : "drop table if exists buildids7_r_sdef;\n"
288 : : "drop table if exists buildids7_file_mtime_scanned;\n"
289 : : "drop table if exists buildids7_files;\n"
290 : : "drop table if exists buildids7_buildids;\n"
291 : : // buildids6: drop bolo/rfolo again, represent sources / rpmcontents in main table
292 : : "drop table if exists buildids6_norm;\n"
293 : : "drop table if exists buildids6_files;\n"
294 : : "drop table if exists buildids6_buildids;\n"
295 : : "drop view if exists buildids6;\n"
296 : : // buildids5: redefine srcfile1 column to be '.'-less (for rpms)
297 : : "drop table if exists buildids5_norm;\n"
298 : : "drop table if exists buildids5_files;\n"
299 : : "drop table if exists buildids5_buildids;\n"
300 : : "drop table if exists buildids5_bolo;\n"
301 : : "drop table if exists buildids5_rfolo;\n"
302 : : "drop view if exists buildids5;\n"
303 : : // buildids4: introduce rpmfile RFOLO
304 : : "drop table if exists buildids4_norm;\n"
305 : : "drop table if exists buildids4_files;\n"
306 : : "drop table if exists buildids4_buildids;\n"
307 : : "drop table if exists buildids4_bolo;\n"
308 : : "drop table if exists buildids4_rfolo;\n"
309 : : "drop view if exists buildids4;\n"
310 : : // buildids3*: split out srcfile BOLO
311 : : "drop table if exists buildids3_norm;\n"
312 : : "drop table if exists buildids3_files;\n"
313 : : "drop table if exists buildids3_buildids;\n"
314 : : "drop table if exists buildids3_bolo;\n"
315 : : "drop view if exists buildids3;\n"
316 : : // buildids2: normalized buildid and filenames into interning tables;
317 : : "drop table if exists buildids2_norm;\n"
318 : : "drop table if exists buildids2_files;\n"
319 : : "drop table if exists buildids2_buildids;\n"
320 : : "drop view if exists buildids2;\n"
321 : : // buildids1: made buildid and artifacttype NULLable, to represent cached-negative
322 : : // lookups from sources, e.g. files or rpms that contain no buildid-indexable content
323 : : "drop table if exists buildids1;\n"
324 : : // buildids: original
325 : : "drop table if exists buildids;\n"
326 : : ;
327 : :
328 : : static const char DEBUGINFOD_SQLITE_CLEANUP_DDL[] =
329 : : "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file
330 : : ;
331 : :
332 : :
333 : :
334 : :
335 : : /* Name and version of program. */
336 : : /* ARGP_PROGRAM_VERSION_HOOK_DEF = print_version; */ // not this simple for C++
337 : :
338 : : /* Bug report address. */
339 : : ARGP_PROGRAM_BUG_ADDRESS_DEF = PACKAGE_BUGREPORT;
340 : :
341 : : /* Definitions of arguments for argp functions. */
342 : : static const struct argp_option options[] =
343 : : {
344 : : { NULL, 0, NULL, 0, "Scanners:", 1 },
345 : : { "scan-file-dir", 'F', NULL, 0, "Enable ELF/DWARF file scanning.", 0 },
346 : : { "scan-rpm-dir", 'R', NULL, 0, "Enable RPM scanning.", 0 },
347 : : { "scan-deb-dir", 'U', NULL, 0, "Enable DEB scanning.", 0 },
348 : : { "scan-archive", 'Z', "EXT=CMD", 0, "Enable arbitrary archive scanning.", 0 },
349 : : // "source-oci-imageregistry" ...
350 : :
351 : : { NULL, 0, NULL, 0, "Options:", 2 },
352 : : { "logical", 'L', NULL, 0, "Follow symlinks, default=ignore.", 0 },
353 : : { "rescan-time", 't', "SECONDS", 0, "Number of seconds to wait between rescans, 0=disable.", 0 },
354 : : { "groom-time", 'g', "SECONDS", 0, "Number of seconds to wait between database grooming, 0=disable.", 0 },
355 : : { "maxigroom", 'G', NULL, 0, "Run a complete database groom/shrink pass at startup.", 0 },
356 : : { "concurrency", 'c', "NUM", 0, "Limit scanning thread concurrency to NUM, default=#CPUs.", 0 },
357 : : { "connection-pool", 'C', "NUM", OPTION_ARG_OPTIONAL,
358 : : "Use webapi connection pool with NUM threads, default=unlim.", 0 },
359 : : { "include", 'I', "REGEX", 0, "Include files matching REGEX, default=all.", 0 },
360 : : { "exclude", 'X', "REGEX", 0, "Exclude files matching REGEX, default=none.", 0 },
361 : : { "port", 'p', "NUM", 0, "HTTP port to listen on, default 8002.", 0 },
362 : : { "database", 'd', "FILE", 0, "Path to sqlite database.", 0 },
363 : : { "ddl", 'D', "SQL", 0, "Apply extra sqlite ddl/pragma to connection.", 0 },
364 : : { "verbose", 'v', NULL, 0, "Increase verbosity.", 0 },
365 : : { "regex-groom", 'r', NULL, 0,"Uses regexes from -I and -X arguments to groom the database.",0},
366 : : #define ARGP_KEY_FDCACHE_FDS 0x1001
367 : : { "fdcache-fds", ARGP_KEY_FDCACHE_FDS, "NUM", 0, "Maximum number of archive files to keep in fdcache.", 0 },
368 : : #define ARGP_KEY_FDCACHE_MBS 0x1002
369 : : { "fdcache-mbs", ARGP_KEY_FDCACHE_MBS, "MB", 0, "Maximum total size of archive file fdcache.", 0 },
370 : : #define ARGP_KEY_FDCACHE_PREFETCH 0x1003
371 : : { "fdcache-prefetch", ARGP_KEY_FDCACHE_PREFETCH, "NUM", 0, "Number of archive files to prefetch into fdcache.", 0 },
372 : : #define ARGP_KEY_FDCACHE_MINTMP 0x1004
373 : : { "fdcache-mintmp", ARGP_KEY_FDCACHE_MINTMP, "NUM", 0, "Minimum free space% on tmpdir.", 0 },
374 : : #define ARGP_KEY_FDCACHE_PREFETCH_MBS 0x1005
375 : : { "fdcache-prefetch-mbs", ARGP_KEY_FDCACHE_PREFETCH_MBS, "MB", 0,"Megabytes allocated to the \
376 : : prefetch cache.", 0},
377 : : #define ARGP_KEY_FDCACHE_PREFETCH_FDS 0x1006
378 : : { "fdcache-prefetch-fds", ARGP_KEY_FDCACHE_PREFETCH_FDS, "NUM", 0,"Number of files allocated to the \
379 : : prefetch cache.", 0},
380 : : #define ARGP_KEY_FORWARDED_TTL_LIMIT 0x1007
381 : : {"forwarded-ttl-limit", ARGP_KEY_FORWARDED_TTL_LIMIT, "NUM", 0, "Limit of X-Forwarded-For hops, default 8.", 0},
382 : : #define ARGP_KEY_PASSIVE 0x1008
383 : : { "passive", ARGP_KEY_PASSIVE, NULL, 0, "Do not scan or groom, read-only database.", 0 },
384 : : { NULL, 0, NULL, 0, NULL, 0 },
385 : : };
386 : :
387 : : /* Short description of program. */
388 : : static const char doc[] = "Serve debuginfo-related content across HTTP from files under PATHs.";
389 : :
390 : : /* Strings for arguments in help texts. */
391 : : static const char args_doc[] = "[PATH ...]";
392 : :
393 : : /* Prototype for option handler. */
394 : : static error_t parse_opt (int key, char *arg, struct argp_state *state);
395 : :
396 : : /* Data structure to communicate with argp functions. */
397 : : static struct argp argp =
398 : : {
399 : : options, parse_opt, args_doc, doc, NULL, NULL, NULL
400 : : };
401 : :
402 : :
403 : : static string db_path;
404 : : static sqlite3 *db; // single connection, serialized across all our threads!
405 : : static sqlite3 *dbq; // webapi query-servicing readonly connection, serialized ditto!
406 : : static unsigned verbose;
407 : : static volatile sig_atomic_t interrupted = 0;
408 : : static volatile sig_atomic_t forced_rescan_count = 0;
409 : : static volatile sig_atomic_t sigusr1 = 0;
410 : : static volatile sig_atomic_t forced_groom_count = 0;
411 : : static volatile sig_atomic_t sigusr2 = 0;
412 : : static unsigned http_port = 8002;
413 : : static unsigned rescan_s = 300;
414 : : static unsigned groom_s = 86400;
415 : : static bool maxigroom = false;
416 : : static unsigned concurrency = std::thread::hardware_concurrency() ?: 1;
417 : : static int connection_pool = 0;
418 : : static set<string> source_paths;
419 : : static bool scan_files = false;
420 : : static map<string,string> scan_archives;
421 : : static vector<string> extra_ddl;
422 : : static regex_t file_include_regex;
423 : : static regex_t file_exclude_regex;
424 : : static bool regex_groom = false;
425 : : static bool traverse_logical;
426 : : static long fdcache_fds;
427 : : static long fdcache_mbs;
428 : : static long fdcache_prefetch;
429 : : static long fdcache_mintmp;
430 : : static long fdcache_prefetch_mbs;
431 : : static long fdcache_prefetch_fds;
432 : : static unsigned forwarded_ttl_limit = 8;
433 : : static string tmpdir;
434 : : static bool passive_p = false;
435 : :
436 : : static void set_metric(const string& key, double value);
437 : : // static void inc_metric(const string& key);
438 : : static void set_metric(const string& metric,
439 : : const string& lname, const string& lvalue,
440 : : double value);
441 : : static void inc_metric(const string& metric,
442 : : const string& lname, const string& lvalue);
443 : : static void add_metric(const string& metric,
444 : : const string& lname, const string& lvalue,
445 : : double value);
446 : : static void inc_metric(const string& metric,
447 : : const string& lname, const string& lvalue,
448 : : const string& rname, const string& rvalue);
449 : : static void add_metric(const string& metric,
450 : : const string& lname, const string& lvalue,
451 : : const string& rname, const string& rvalue,
452 : : double value);
453 : :
454 : :
455 : : class tmp_inc_metric { // a RAII style wrapper for exception-safe scoped increment & decrement
456 : : string m, n, v;
457 : : public:
458 : 901 : tmp_inc_metric(const string& mname, const string& lname, const string& lvalue):
459 [ + - + - : 901 : m(mname), n(lname), v(lvalue)
- - - - -
- ]
460 : : {
461 [ + - ]: 901 : add_metric (m, n, v, 1);
462 : 901 : }
463 : 901 : ~tmp_inc_metric()
464 [ - + - + : 901 : {
- + ]
465 : 901 : add_metric (m, n, v, -1);
466 : 901 : }
467 : : };
468 : :
469 : : class tmp_ms_metric { // a RAII style wrapper for exception-safe scoped timing
470 : : string m, n, v;
471 : : struct timespec ts_start;
472 : : public:
473 : 15872 : tmp_ms_metric(const string& mname, const string& lname, const string& lvalue):
474 [ + - + - : 15872 : m(mname), n(lname), v(lvalue)
- - - - ]
475 : : {
476 : 15869 : clock_gettime (CLOCK_MONOTONIC, & ts_start);
477 : 15869 : }
478 : 15867 : ~tmp_ms_metric()
479 [ - + - + ]: 15872 : {
480 : 15867 : struct timespec ts_end;
481 : 15867 : clock_gettime (CLOCK_MONOTONIC, & ts_end);
482 : 15871 : double deltas = (ts_end.tv_sec - ts_start.tv_sec)
483 : 15871 : + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
484 : :
485 [ + - ]: 15871 : add_metric (m + "_milliseconds_sum", n, v, (deltas*1000.0));
486 [ + - + + ]: 31744 : inc_metric (m + "_milliseconds_count", n, v);
487 : 15872 : }
488 : : };
489 : :
490 : :
491 : : /* Handle program arguments. */
492 : : static error_t
493 : 479 : parse_opt (int key, char *arg,
494 : : struct argp_state *state __attribute__ ((unused)))
495 : : {
496 : 479 : int rc;
497 [ + + + + : 479 : switch (key)
+ + + + -
+ + - - +
+ + + + +
- + + + +
+ + + ]
498 : : {
499 : 115 : case 'v': verbose ++; break;
500 : 32 : case 'd':
501 : : /* When using the in-memory database make sure it is shareable,
502 : : so we can open it twice as read/write and read-only. */
503 [ + + ]: 32 : if (strcmp (arg, ":memory:") == 0)
504 : 479 : db_path = "file::memory:?cache=shared";
505 : : else
506 [ + - ]: 50 : db_path = string(arg);
507 : : break;
508 : 32 : case 'p': http_port = (unsigned) atoi(arg);
509 [ - + ]: 32 : if (http_port == 0 || http_port > 65535)
510 : 0 : argp_failure(state, 1, EINVAL, "port number");
511 : : break;
512 : 23 : case 'F': scan_files = true; break;
513 : 7 : case 'R':
514 [ + - + - : 7 : scan_archives[".rpm"]="cat"; // libarchive groks rpm natively
- + ]
515 : 7 : break;
516 : 7 : case 'U':
517 [ + - + - : 7 : scan_archives[".deb"]="(bsdtar -O -x -f - data.tar\\*)<";
- + ]
518 [ + - + - : 7 : scan_archives[".ddeb"]="(bsdtar -O -x -f - data.tar\\*)<";
- + ]
519 [ + - + - : 7 : scan_archives[".ipk"]="(bsdtar -O -x -f - data.tar\\*)<";
- + ]
520 : : // .udeb too?
521 : 7 : break;
522 : 14 : case 'Z':
523 : 14 : {
524 [ - + ]: 14 : char* extension = strchr(arg, '=');
525 [ - + ]: 14 : if (arg[0] == '\0')
526 : 0 : argp_failure(state, 1, EINVAL, "missing EXT");
527 [ + + ]: 14 : else if (extension)
528 [ + - + - : 7 : scan_archives[string(arg, (extension-arg))]=string(extension+1);
- + - + ]
529 : : else
530 [ + - + - : 7 : scan_archives[string(arg)]=string("cat");
- + - + ]
531 : : }
532 : : break;
533 : 4 : case 'L':
534 [ - + ]: 4 : if (passive_p)
535 : 0 : argp_failure(state, 1, EINVAL, "-L option inconsistent with passive mode");
536 : 4 : traverse_logical = true;
537 : 4 : break;
538 : 0 : case 'D':
539 [ # # ]: 0 : if (passive_p)
540 : 0 : argp_failure(state, 1, EINVAL, "-D option inconsistent with passive mode");
541 [ # # ]: 0 : extra_ddl.push_back(string(arg));
542 : 0 : break;
543 : 24 : case 't':
544 [ - + ]: 24 : if (passive_p)
545 : 0 : argp_failure(state, 1, EINVAL, "-t option inconsistent with passive mode");
546 : 24 : rescan_s = (unsigned) atoi(arg);
547 : 24 : break;
548 : 24 : case 'g':
549 [ - + ]: 24 : if (passive_p)
550 : 0 : argp_failure(state, 1, EINVAL, "-g option inconsistent with passive mode");
551 : 24 : groom_s = (unsigned) atoi(arg);
552 : 24 : break;
553 : 0 : case 'G':
554 [ # # ]: 0 : if (passive_p)
555 : 0 : argp_failure(state, 1, EINVAL, "-G option inconsistent with passive mode");
556 : 0 : maxigroom = true;
557 : 0 : break;
558 : 0 : case 'c':
559 [ # # ]: 0 : if (passive_p)
560 : 0 : argp_failure(state, 1, EINVAL, "-c option inconsistent with passive mode");
561 : 0 : concurrency = (unsigned) atoi(arg);
562 [ # # ]: 0 : if (concurrency < 1) concurrency = 1;
563 : : break;
564 : 3 : case 'C':
565 [ + + ]: 3 : if (arg)
566 : : {
567 : 2 : connection_pool = atoi(arg);
568 [ - + ]: 2 : if (connection_pool < 2)
569 : 0 : argp_failure(state, 1, EINVAL, "-C NUM minimum 2");
570 : : }
571 : : else // arg not given
572 [ + - ]: 2 : connection_pool = std::thread::hardware_concurrency() * 2 ?: 2;
573 : : break;
574 : 1 : case 'I':
575 : : // NB: no problem with unconditional free here - an earlier failed regcomp would exit program
576 [ - + ]: 1 : if (passive_p)
577 : 0 : argp_failure(state, 1, EINVAL, "-I option inconsistent with passive mode");
578 : 1 : regfree (&file_include_regex);
579 : 1 : rc = regcomp (&file_include_regex, arg, REG_EXTENDED|REG_NOSUB);
580 [ - + ]: 1 : if (rc != 0)
581 : 0 : argp_failure(state, 1, EINVAL, "regular expression");
582 : : break;
583 : 1 : case 'X':
584 [ - + ]: 1 : if (passive_p)
585 : 0 : argp_failure(state, 1, EINVAL, "-X option inconsistent with passive mode");
586 : 1 : regfree (&file_exclude_regex);
587 : 1 : rc = regcomp (&file_exclude_regex, arg, REG_EXTENDED|REG_NOSUB);
588 [ - + ]: 1 : if (rc != 0)
589 : 0 : argp_failure(state, 1, EINVAL, "regular expression");
590 : : break;
591 : 1 : case 'r':
592 [ - + ]: 1 : if (passive_p)
593 : 0 : argp_failure(state, 1, EINVAL, "-r option inconsistent with passive mode");
594 : 1 : regex_groom = true;
595 : 1 : break;
596 : 5 : case ARGP_KEY_FDCACHE_FDS:
597 : 5 : fdcache_fds = atol (arg);
598 : 5 : break;
599 : 1 : case ARGP_KEY_FDCACHE_MBS:
600 : 1 : fdcache_mbs = atol (arg);
601 : 1 : break;
602 : 0 : case ARGP_KEY_FDCACHE_PREFETCH:
603 : 0 : fdcache_prefetch = atol (arg);
604 : 0 : break;
605 : 1 : case ARGP_KEY_FDCACHE_MINTMP:
606 : 1 : fdcache_mintmp = atol (arg);
607 [ - + ]: 1 : if( fdcache_mintmp > 100 || fdcache_mintmp < 0 )
608 : 0 : argp_failure(state, 1, EINVAL, "fdcache mintmp percent");
609 : : break;
610 : 2 : case ARGP_KEY_FORWARDED_TTL_LIMIT:
611 : 2 : forwarded_ttl_limit = (unsigned) atoi(arg);
612 : 2 : break;
613 : 42 : case ARGP_KEY_ARG:
614 [ + - ]: 84 : source_paths.insert(string(arg));
615 : 42 : break;
616 : 5 : case ARGP_KEY_FDCACHE_PREFETCH_FDS:
617 : 5 : fdcache_prefetch_fds = atol(arg);
618 [ - + ]: 5 : if ( fdcache_prefetch_fds < 0)
619 : 0 : argp_failure(state, 1, EINVAL, "fdcache prefetch fds");
620 : : break;
621 : 1 : case ARGP_KEY_FDCACHE_PREFETCH_MBS:
622 : 1 : fdcache_prefetch_mbs = atol(arg);
623 [ - + ]: 1 : if ( fdcache_prefetch_mbs < 0)
624 : 0 : argp_failure(state, 1, EINVAL, "fdcache prefetch mbs");
625 : : break;
626 : 1 : case ARGP_KEY_PASSIVE:
627 : 1 : passive_p = true;
628 [ + - ]: 1 : if (source_paths.size() > 0
629 [ + - ]: 1 : || maxigroom
630 [ + - ]: 1 : || extra_ddl.size() > 0
631 [ + - - + ]: 2 : || traverse_logical)
632 : : // other conflicting options tricky to check
633 : 0 : argp_failure(state, 1, EINVAL, "inconsistent options with passive mode");
634 : : break;
635 : : // case 'h': argp_state_help (state, stderr, ARGP_HELP_LONG|ARGP_HELP_EXIT_OK);
636 : : default: return ARGP_ERR_UNKNOWN;
637 : : }
638 : :
639 : : return 0;
640 : : }
641 : :
642 : :
643 : : ////////////////////////////////////////////////////////////////////////
644 : :
645 : :
646 : : static void add_mhd_response_header (struct MHD_Response *r,
647 : : const char *h, const char *v);
648 : :
649 : : // represent errors that may get reported to an ostream and/or a libmicrohttpd connection
650 : :
651 : 2 : struct reportable_exception
652 : : {
653 : : int code;
654 : : string message;
655 : :
656 [ - - + - : 127 : reportable_exception(int c, const string& m): code(c), message(m) {}
+ - + - ]
657 [ - - + - : 146 : reportable_exception(const string& m): code(503), message(m) {}
- - + - -
- + - - -
- - - - +
- - - ]
658 : : reportable_exception(): code(503), message() {}
659 : :
660 : : void report(ostream& o) const; // defined under obatched() class below
661 : :
662 : 243 : MHD_RESULT mhd_send_response(MHD_Connection* c) const {
663 : 243 : MHD_Response* r = MHD_create_response_from_buffer (message.size(),
664 : 243 : (void*) message.c_str(),
665 : : MHD_RESPMEM_MUST_COPY);
666 : 243 : add_mhd_response_header (r, "Content-Type", "text/plain");
667 : 243 : MHD_RESULT rc = MHD_queue_response (c, code, r);
668 : 243 : MHD_destroy_response (r);
669 : 243 : return rc;
670 : : }
671 : : };
672 : :
673 : :
674 : : struct sqlite_exception: public reportable_exception
675 : : {
676 : 0 : sqlite_exception(int rc, const string& msg):
677 [ # # # # : 0 : reportable_exception(string("sqlite3 error: ") + msg + ": " + string(sqlite3_errstr(rc) ?: "?")) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
678 [ # # # # : 0 : inc_metric("error_count","sqlite3",sqlite3_errstr(rc));
# # # # #
# # # # #
# # # # #
# ]
679 : 0 : }
680 : : };
681 : :
682 [ + - - - ]: 1 : struct libc_exception: public reportable_exception
683 : : {
684 : 140 : libc_exception(int rc, const string& msg):
685 [ + - + - : 560 : reportable_exception(string("libc error: ") + msg + ": " + string(strerror(rc) ?: "?")) {
- + + - +
- + - + -
- + - + -
+ + - - -
- - - - -
- ]
686 [ + - + - : 280 : inc_metric("error_count","libc",strerror(rc));
+ - + - -
+ - + - -
- - - - ]
687 : 140 : }
688 : : };
689 : :
690 : :
691 : : struct archive_exception: public reportable_exception
692 : : {
693 : 0 : archive_exception(const string& msg):
694 [ # # # # : 0 : reportable_exception(string("libarchive error: ") + msg) {
# # # # #
# ]
695 [ # # # # : 0 : inc_metric("error_count","libarchive",msg);
# # # # #
# # # ]
696 : 0 : }
697 : 0 : archive_exception(struct archive* a, const string& msg):
698 [ # # # # : 0 : reportable_exception(string("libarchive error: ") + msg + ": " + string(archive_error_string(a) ?: "?")) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
699 [ # # # # : 0 : inc_metric("error_count","libarchive",msg + ": " + string(archive_error_string(a) ?: "?"));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
700 : 0 : }
701 : : };
702 : :
703 : :
704 : : struct elfutils_exception: public reportable_exception
705 : : {
706 : 0 : elfutils_exception(int rc, const string& msg):
707 [ # # # # : 0 : reportable_exception(string("elfutils error: ") + msg + ": " + string(elf_errmsg(rc) ?: "?")) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
708 [ # # # # : 0 : inc_metric("error_count","elfutils",elf_errmsg(rc));
# # # # #
# # # # #
# # # # #
# ]
709 : 0 : }
710 : : };
711 : :
712 : :
713 : : ////////////////////////////////////////////////////////////////////////
714 : :
715 : : template <typename Payload>
716 : : class workq
717 : : {
718 : : set<Payload> q; // eliminate duplicates
719 : : mutex mtx;
720 : : condition_variable cv;
721 : : bool dead;
722 : : unsigned idlers; // number of threads busy with wait_idle / done_idle
723 : : unsigned fronters; // number of threads busy with wait_front / done_front
724 : :
725 : : public:
726 : 32 : workq() { dead = false; idlers = 0; fronters = 0; }
727 : 32 : ~workq() {}
728 : :
729 : 366 : void push_back(const Payload& p)
730 : : {
731 : 366 : unique_lock<mutex> lock(mtx);
732 [ + - ]: 366 : q.insert (p);
733 [ + - + - : 1098 : set_metric("thread_work_pending","role","scan", q.size());
+ - + - -
+ + - - -
- - - - ]
734 [ + - ]: 366 : cv.notify_all();
735 : 366 : }
736 : :
737 : : // kill this workqueue, wake up all idlers / scanners
738 : 32 : void nuke() {
739 : 32 : unique_lock<mutex> lock(mtx);
740 : : // optional: q.clear();
741 : 32 : dead = true;
742 [ + - ]: 32 : cv.notify_all();
743 : 32 : }
744 : :
745 : : // clear the workqueue, when scanning is interrupted with USR2
746 : 0 : void clear() {
747 : 0 : unique_lock<mutex> lock(mtx);
748 [ # # ]: 0 : q.clear();
749 [ # # # # : 0 : set_metric("thread_work_pending","role","scan", q.size());
# # # # #
# # # # #
# # # # ]
750 : : // NB: there may still be some live fronters
751 [ # # ]: 0 : cv.notify_all(); // maybe wake up waiting idlers
752 : 0 : }
753 : :
754 : : // block this scanner thread until there is work to do and no active idler
755 : 478 : bool wait_front (Payload& p)
756 : : {
757 : 956 : unique_lock<mutex> lock(mtx);
758 [ + + + + : 2680 : while (!dead && (q.size() == 0 || idlers > 0))
+ + ]
759 : 2202 : cv.wait(lock);
760 [ + + ]: 478 : if (dead)
761 : : return false;
762 : : else
763 : : {
764 [ + - ]: 366 : p = * q.begin();
765 : 366 : q.erase (q.begin());
766 [ + - ]: 366 : fronters ++; // prevent idlers from starting awhile, even if empty q
767 [ + - + - : 1098 : set_metric("thread_work_pending","role","scan", q.size());
+ - + - -
+ + - - -
- - - - ]
768 : : // NB: don't wake up idlers yet! The consumer is busy
769 : : // processing this element until it calls done_front().
770 : 366 : return true;
771 : : }
772 : : }
773 : :
774 : : // notify waitq that scanner thread is done with that last item
775 : 366 : void done_front ()
776 : : {
777 : 366 : unique_lock<mutex> lock(mtx);
778 : 366 : fronters --;
779 [ + + + + ]: 366 : if (q.size() == 0 && fronters == 0)
780 : 52 : cv.notify_all(); // maybe wake up waiting idlers
781 : 366 : }
782 : :
783 : : // block this idler thread until there is no work to do
784 : 306 : void wait_idle ()
785 : : {
786 : 306 : unique_lock<mutex> lock(mtx);
787 : 306 : cv.notify_all(); // maybe wake up waiting scanners
788 [ + + + + : 327 : while (!dead && ((q.size() != 0) || fronters > 0))
+ + ]
789 : 21 : cv.wait(lock);
790 [ + - ]: 306 : idlers ++;
791 : 306 : }
792 : :
793 : 275 : void done_idle ()
794 : : {
795 : 275 : unique_lock<mutex> lock(mtx);
796 : 275 : idlers --;
797 [ + - ]: 275 : cv.notify_all(); // maybe wake up waiting scanners, but probably not (shutting down)
798 : 275 : }
799 : : };
800 : :
801 : : typedef struct stat stat_t;
802 : : typedef pair<string,stat_t> scan_payload;
803 : 863 : inline bool operator< (const scan_payload& a, const scan_payload& b)
804 : : {
805 [ + + + + : 863 : return a.first < b.first; // don't bother compare the stat fields
+ - ]
806 : : }
807 : : static workq<scan_payload> scanq; // just a single one
808 : : // producer & idler: thread_main_fts_source_paths()
809 : : // consumer: thread_main_scanner()
810 : : // idler: thread_main_groom()
811 : :
812 : :
813 : : ////////////////////////////////////////////////////////////////////////
814 : :
815 : : // Unique set is a thread-safe structure that lends 'ownership' of a value
816 : : // to a thread. Other threads requesting the same thing are made to wait.
817 : : // It's like a semaphore-on-demand.
818 : : template <typename T>
819 : : class unique_set
820 : : {
821 : : private:
822 : : set<T> values;
823 : : mutex mtx;
824 : : condition_variable cv;
825 : : public:
826 : 25 : unique_set() {}
827 : 25 : ~unique_set() {}
828 : :
829 : 593 : void acquire(const T& value)
830 : : {
831 : 593 : unique_lock<mutex> lock(mtx);
832 [ + + ]: 5306 : while (values.find(value) != values.end())
833 : 4712 : cv.wait(lock);
834 [ + - ]: 594 : values.insert(value);
835 : 594 : }
836 : :
837 : 594 : void release(const T& value)
838 : : {
839 : 594 : unique_lock<mutex> lock(mtx);
840 : : // assert (values.find(value) != values.end());
841 : 594 : values.erase(value);
842 [ + - ]: 594 : cv.notify_all();
843 : 594 : }
844 : : };
845 : :
846 : :
847 : : // This is the object that's instantiate to uniquely hold a value in a
848 : : // RAII-pattern way.
849 : : template <typename T>
850 : : class unique_set_reserver
851 : : {
852 : : private:
853 : : unique_set<T>& please_hold;
854 : : T mine;
855 : : public:
856 : 594 : unique_set_reserver(unique_set<T>& t, const T& value):
857 [ + - - - ]: 594 : please_hold(t), mine(value) { please_hold.acquire(mine); }
858 [ + - ]: 594 : ~unique_set_reserver() { please_hold.release(mine); }
859 : : };
860 : :
861 : :
862 : : ////////////////////////////////////////////////////////////////////////
863 : :
864 : :
865 : : // Print a standard timestamp.
866 : : static ostream&
867 : 7825 : timestamp (ostream &o)
868 : : {
869 : 7825 : char datebuf[80];
870 : 7825 : char *now2 = NULL;
871 : 7825 : time_t now_t = time(NULL);
872 : 7825 : struct tm now;
873 : 7825 : struct tm *nowp = gmtime_r (&now_t, &now);
874 [ + - ]: 7825 : if (nowp)
875 : : {
876 : 7825 : (void) strftime (datebuf, sizeof (datebuf), "%c", nowp);
877 : 7825 : now2 = datebuf;
878 : : }
879 : :
880 : 7825 : return o << "[" << (now2 ? now2 : "") << "] "
881 [ - + ]: 7825 : << "(" << getpid () << "/" << tid() << "): ";
882 : : }
883 : :
884 : :
885 : : // A little class that impersonates an ostream to the extent that it can
886 : : // take << streaming operations. It batches up the bits into an internal
887 : : // stringstream until it is destroyed; then flushes to the original ostream.
888 : : // It adds a timestamp
889 : : class obatched
890 : : {
891 : : private:
892 : : ostream& o;
893 : : stringstream stro;
894 : : static mutex lock;
895 : : public:
896 : 7825 : obatched(ostream& oo, bool timestamp_p = true): o(oo)
897 : : {
898 [ + - ]: 7825 : if (timestamp_p)
899 [ + - ]: 7825 : timestamp(stro);
900 : 7823 : }
901 : 7822 : ~obatched()
902 : 15650 : {
903 : 15647 : unique_lock<mutex> do_not_cross_the_streams(obatched::lock);
904 [ + - ]: 7825 : o << stro.str();
905 [ + - ]: 7825 : o.flush();
906 : 7825 : }
907 : : operator ostream& () { return stro; }
908 [ - - + - : 7139 : template <typename T> ostream& operator << (const T& t) { stro << t; return stro; }
+ - + - +
- + - - -
- - - - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - - - +
- + - + -
+ - + - -
- + - + -
+ - + - +
- + - + -
+ - + - -
- + - + -
+ - + - -
- - - - -
+ - - - +
- + - + -
+ - + - -
- - - +
- ]
909 : : };
910 : : mutex obatched::lock; // just the one, since cout/cerr iostreams are not thread-safe
911 : :
912 : :
913 : 273 : void reportable_exception::report(ostream& o) const {
914 [ + - + - ]: 273 : obatched(o) << message << endl;
915 : 273 : }
916 : :
917 : :
918 : : ////////////////////////////////////////////////////////////////////////
919 : :
920 : :
921 : : // RAII style sqlite prepared-statement holder that matches { } block lifetime
922 : :
923 : : struct sqlite_ps
924 : : {
925 : : private:
926 : : sqlite3* db;
927 : : const string nickname;
928 : : const string sql;
929 : : sqlite3_stmt *pp;
930 : :
931 : : sqlite_ps(const sqlite_ps&); // make uncopyable
932 : : sqlite_ps& operator=(const sqlite_ps &); // make unassignable
933 : :
934 : : public:
935 [ + - - - ]: 2333 : sqlite_ps (sqlite3* d, const string& n, const string& s): db(d), nickname(n), sql(s) {
936 : : // tmp_ms_metric tick("sqlite3","prep",nickname);
937 [ - + ]: 2333 : if (verbose > 4)
938 [ # # # # : 0 : obatched(clog) << nickname << " prep " << sql << endl;
# # # # #
# # # ]
939 [ + - ]: 2333 : int rc = sqlite3_prepare_v2 (db, sql.c_str(), -1 /* to \0 */, & this->pp, NULL);
940 [ - + ]: 2333 : if (rc != SQLITE_OK)
941 [ # # # # ]: 0 : throw sqlite_exception(rc, "prepare " + sql);
942 : 2333 : }
943 : :
944 : 8156 : sqlite_ps& reset()
945 : : {
946 [ + - + - : 16312 : tmp_ms_metric tick("sqlite3","reset",nickname);
- + + - -
- ]
947 [ + - ]: 8156 : sqlite3_reset(this->pp);
948 : 8155 : return *this;
949 : : }
950 : :
951 : 10663 : sqlite_ps& bind(int parameter, const string& str)
952 : : {
953 [ - + ]: 10663 : if (verbose > 4)
954 [ # # # # : 0 : obatched(clog) << nickname << " bind " << parameter << "=" << str << endl;
# # # # #
# # # ]
955 : 10663 : int rc = sqlite3_bind_text (this->pp, parameter, str.c_str(), -1, SQLITE_TRANSIENT);
956 [ - + ]: 10665 : if (rc != SQLITE_OK)
957 [ # # # # ]: 0 : throw sqlite_exception(rc, "sqlite3 bind");
958 : 10665 : return *this;
959 : : }
960 : :
961 : 3226 : sqlite_ps& bind(int parameter, int64_t value)
962 : : {
963 [ - + ]: 3226 : if (verbose > 4)
964 [ # # # # : 0 : obatched(clog) << nickname << " bind " << parameter << "=" << value << endl;
# # # # #
# # # ]
965 : 3226 : int rc = sqlite3_bind_int64 (this->pp, parameter, value);
966 [ - + ]: 3224 : if (rc != SQLITE_OK)
967 [ # # # # ]: 0 : throw sqlite_exception(rc, "sqlite3 bind");
968 : 3224 : return *this;
969 : : }
970 : :
971 : : sqlite_ps& bind(int parameter)
972 : : {
973 : : if (verbose > 4)
974 : : obatched(clog) << nickname << " bind " << parameter << "=" << "NULL" << endl;
975 : : int rc = sqlite3_bind_null (this->pp, parameter);
976 : : if (rc != SQLITE_OK)
977 : : throw sqlite_exception(rc, "sqlite3 bind");
978 : : return *this;
979 : : }
980 : :
981 : :
982 : 6089 : void step_ok_done() {
983 [ + - + - : 18265 : tmp_ms_metric tick("sqlite3","step_done",nickname);
- + + - -
- ]
984 [ + - ]: 6087 : int rc = sqlite3_step (this->pp);
985 [ - + ]: 6089 : if (verbose > 4)
986 [ # # # # : 0 : obatched(clog) << nickname << " step-ok-done(" << sqlite3_errstr(rc) << ") " << sql << endl;
# # # # #
# # # # #
# # ]
987 [ + + - + ]: 6089 : if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)
988 [ # # # # ]: 0 : throw sqlite_exception(rc, "sqlite3 step");
989 [ + - ]: 6089 : (void) sqlite3_reset (this->pp);
990 : 6089 : }
991 : :
992 : :
993 : 1627 : int step() {
994 [ + - + - : 3254 : tmp_ms_metric tick("sqlite3","step",nickname);
- + + - -
- ]
995 [ + - ]: 1627 : int rc = sqlite3_step (this->pp);
996 [ - + ]: 1627 : if (verbose > 4)
997 [ # # # # : 0 : obatched(clog) << nickname << " step(" << sqlite3_errstr(rc) << ") " << sql << endl;
# # # # #
# # # # #
# # ]
998 : 1627 : return rc;
999 : : }
1000 : :
1001 [ + + + + ]: 4635 : ~sqlite_ps () { sqlite3_finalize (this->pp); }
1002 : 2942 : operator sqlite3_stmt* () { return this->pp; }
1003 : : };
1004 : :
1005 : :
1006 : : ////////////////////////////////////////////////////////////////////////
1007 : :
1008 : : // RAII style templated autocloser
1009 : :
1010 : : template <class Payload, class Ignore>
1011 : : struct defer_dtor
1012 : : {
1013 : : public:
1014 : : typedef Ignore (*dtor_fn) (Payload);
1015 : :
1016 : : private:
1017 : : Payload p;
1018 : : dtor_fn fn;
1019 : :
1020 : : public:
1021 : 2036 : defer_dtor(Payload _p, dtor_fn _fn): p(_p), fn(_fn) {}
1022 : 718 : ~defer_dtor() { (void) (*fn)(p); }
1023 : :
1024 : : private:
1025 : : defer_dtor(const defer_dtor<Payload,Ignore>&); // make uncopyable
1026 : : defer_dtor& operator=(const defer_dtor<Payload,Ignore> &); // make unassignable
1027 : : };
1028 : :
1029 : :
1030 : :
1031 : : ////////////////////////////////////////////////////////////////////////
1032 : :
1033 : :
1034 : : static string
1035 : 1808 : header_censor(const string& str)
1036 : : {
1037 : 1808 : string y;
1038 [ + + ]: 15229 : for (auto&& x : str)
1039 : : {
1040 [ + + + + : 13421 : if (isalnum(x) || x == '/' || x == '.' || x == ',' || x == '_' || x == ':')
+ + + + +
- - + ]
1041 [ + - ]: 26839 : y += x;
1042 : : }
1043 : 1808 : return y;
1044 : : }
1045 : :
1046 : :
1047 : : static string
1048 : 904 : conninfo (struct MHD_Connection * conn)
1049 : : {
1050 : 904 : char hostname[256]; // RFC1035
1051 : 904 : char servname[256];
1052 : 904 : int sts = -1;
1053 : :
1054 [ - + ]: 904 : if (conn == 0)
1055 : 0 : return "internal";
1056 : :
1057 : : /* Look up client address data. */
1058 : 904 : const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
1059 : : MHD_CONNECTION_INFO_CLIENT_ADDRESS);
1060 [ + - ]: 904 : struct sockaddr *so = u ? u->client_addr : 0;
1061 : :
1062 [ + - - + ]: 904 : if (so && so->sa_family == AF_INET) {
1063 : 0 : sts = getnameinfo (so, sizeof (struct sockaddr_in), hostname, sizeof (hostname), servname,
1064 : : sizeof (servname), NI_NUMERICHOST | NI_NUMERICSERV);
1065 [ + - + - ]: 904 : } else if (so && so->sa_family == AF_INET6) {
1066 : 904 : struct sockaddr_in6* addr6 = (struct sockaddr_in6*) so;
1067 [ + - + - : 904 : if (IN6_IS_ADDR_V4MAPPED(&addr6->sin6_addr)) {
+ + ]
1068 : 642 : struct sockaddr_in addr4;
1069 : 642 : memset (&addr4, 0, sizeof(addr4));
1070 : 642 : addr4.sin_family = AF_INET;
1071 : 642 : addr4.sin_port = addr6->sin6_port;
1072 : 642 : memcpy (&addr4.sin_addr.s_addr, addr6->sin6_addr.s6_addr+12, sizeof(addr4.sin_addr.s_addr));
1073 : 642 : sts = getnameinfo ((struct sockaddr*) &addr4, sizeof (addr4),
1074 : : hostname, sizeof (hostname), servname, sizeof (servname),
1075 : : NI_NUMERICHOST | NI_NUMERICSERV);
1076 : : } else {
1077 : 262 : sts = getnameinfo (so, sizeof (struct sockaddr_in6), hostname, sizeof (hostname), NULL, 0,
1078 : : NI_NUMERICHOST);
1079 : : }
1080 : : }
1081 : :
1082 [ - + ]: 904 : if (sts != 0) {
1083 : 0 : hostname[0] = servname[0] = '\0';
1084 : : }
1085 : :
1086 : : // extract headers relevant to administration
1087 [ - + ]: 904 : const char* user_agent = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";
1088 [ + + ]: 904 : const char* x_forwarded_for = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";
1089 : : // NB: these are untrustworthy, beware if machine-processing log files
1090 : :
1091 [ + - + - : 2712 : return string(hostname) + string(":") + string(servname) +
+ - + - -
+ - + - +
- + - + -
- - - - -
- - - - ]
1092 [ + - + - : 3741 : string(" UA:") + header_censor(string(user_agent)) +
+ - + - +
- - + - +
+ + + + -
+ - - - -
- - - - -
- ]
1093 [ + - + - : 2715 : string(" XFF:") + header_censor(string(x_forwarded_for));
+ - + - +
- + + - +
- + - - -
- - - ]
1094 : : }
1095 : :
1096 : :
1097 : :
1098 : : ////////////////////////////////////////////////////////////////////////
1099 : :
1100 : : /* Wrapper for MHD_add_response_header that logs an error if we
1101 : : couldn't add the specified header. */
1102 : : static void
1103 : 2738 : add_mhd_response_header (struct MHD_Response *r,
1104 : : const char *h, const char *v)
1105 : : {
1106 [ - + ]: 2738 : if (MHD_add_response_header (r, h, v) == MHD_NO)
1107 [ # # # # : 0 : obatched(clog) << "Error: couldn't add '" << h << "' header" << endl;
# # # # ]
1108 : 2738 : }
1109 : :
1110 : : static void
1111 : 371 : add_mhd_last_modified (struct MHD_Response *resp, time_t mtime)
1112 : : {
1113 : 371 : struct tm now;
1114 : 371 : struct tm *nowp = gmtime_r (&mtime, &now);
1115 [ + - ]: 371 : if (nowp != NULL)
1116 : : {
1117 : 371 : char datebuf[80];
1118 : 371 : size_t rc = strftime (datebuf, sizeof (datebuf), "%a, %d %b %Y %T GMT",
1119 : : nowp);
1120 [ + - ]: 371 : if (rc > 0 && rc < sizeof (datebuf))
1121 : 371 : add_mhd_response_header (resp, "Last-Modified", datebuf);
1122 : : }
1123 : :
1124 : 371 : add_mhd_response_header (resp, "Cache-Control", "public");
1125 : 371 : }
1126 : :
1127 : :
1128 : :
1129 : : static struct MHD_Response*
1130 : 32 : handle_buildid_f_match (bool internal_req_t,
1131 : : int64_t b_mtime,
1132 : : const string& b_source0,
1133 : : int *result_fd)
1134 : : {
1135 : 32 : (void) internal_req_t; // ignored
1136 : 32 : int fd = open(b_source0.c_str(), O_RDONLY);
1137 [ - + ]: 32 : if (fd < 0)
1138 [ # # # # : 0 : throw libc_exception (errno, string("open ") + b_source0);
# # # # #
# ]
1139 : :
1140 : : // NB: use manual close(2) in error case instead of defer_dtor, because
1141 : : // in the normal case, we want to hand the fd over to libmicrohttpd for
1142 : : // file transfer.
1143 : :
1144 : 32 : struct stat s;
1145 : 32 : int rc = fstat(fd, &s);
1146 [ - + ]: 32 : if (rc < 0)
1147 : : {
1148 : 0 : close(fd);
1149 [ # # # # : 0 : throw libc_exception (errno, string("fstat ") + b_source0);
# # # # #
# ]
1150 : : }
1151 : :
1152 [ - + ]: 32 : if ((int64_t) s.st_mtime != b_mtime)
1153 : : {
1154 [ # # ]: 0 : if (verbose)
1155 [ # # # # ]: 0 : obatched(clog) << "mtime mismatch for " << b_source0 << endl;
1156 : 0 : close(fd);
1157 : 0 : return 0;
1158 : : }
1159 : :
1160 [ + - + - : 96 : inc_metric ("http_responses_total","result","file");
+ - - + +
- - - -
- ]
1161 : 32 : struct MHD_Response* r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd);
1162 [ - + ]: 32 : if (r == 0)
1163 : : {
1164 [ # # ]: 0 : if (verbose)
1165 [ # # # # ]: 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
1166 : 0 : close(fd);
1167 : : }
1168 : : else
1169 : : {
1170 : 64 : std::string file = b_source0.substr(b_source0.find_last_of("/")+1, b_source0.length());
1171 [ + - ]: 32 : add_mhd_response_header (r, "Content-Type", "application/octet-stream");
1172 [ + - ]: 32 : add_mhd_response_header (r, "X-DEBUGINFOD-SIZE",
1173 [ + - ]: 32 : to_string(s.st_size).c_str());
1174 [ + - ]: 32 : add_mhd_response_header (r, "X-DEBUGINFOD-FILE", file.c_str());
1175 [ + - ]: 32 : add_mhd_last_modified (r, s.st_mtime);
1176 [ + - ]: 32 : if (verbose > 1)
1177 [ + - + - : 64 : obatched(clog) << "serving file " << b_source0 << endl;
+ - - - ]
1178 : : /* libmicrohttpd will close it. */
1179 [ + - ]: 32 : if (result_fd)
1180 : 32 : *result_fd = fd;
1181 : : }
1182 : :
1183 : : return r;
1184 : : }
1185 : :
1186 : :
1187 : : // quote all questionable characters of str for safe passage through a sh -c expansion.
1188 : : static string
1189 : 277 : shell_escape(const string& str)
1190 : : {
1191 : 277 : string y;
1192 [ + + ]: 27378 : for (auto&& x : str)
1193 : : {
1194 [ + + + + ]: 27101 : if (! isalnum(x) && x != '/')
1195 [ + - ]: 2998 : y += "\\";
1196 [ + - ]: 54202 : y += x;
1197 : : }
1198 : 277 : return y;
1199 : : }
1200 : :
1201 : :
1202 : : // PR25548: Perform POSIX / RFC3986 style path canonicalization on the input string.
1203 : : //
1204 : : // Namely:
1205 : : // // -> /
1206 : : // /foo/../ -> /
1207 : : // /./ -> /
1208 : : //
1209 : : // This mapping is done on dwarf-side source path names, which may
1210 : : // include these constructs, so we can deal with debuginfod clients
1211 : : // that accidentally canonicalize the paths.
1212 : : //
1213 : : // realpath(3) is close but not quite right, because it also resolves
1214 : : // symbolic links. Symlinks at the debuginfod server have nothing to
1215 : : // do with the build-time symlinks, thus they must not be considered.
1216 : : //
1217 : : // see also curl Curl_dedotdotify() aka RFC3986, which we mostly follow here
1218 : : // see also libc __realpath()
1219 : : // see also llvm llvm::sys::path::remove_dots()
1220 : : static string
1221 : 1643 : canon_pathname (const string& input)
1222 : : {
1223 : 1643 : string i = input; // 5.2.4 (1)
1224 : 1643 : string o;
1225 : :
1226 : 10476 : while (i.size() != 0)
1227 : : {
1228 : : // 5.2.4 (2) A
1229 [ + - - + : 17666 : if (i.substr(0,3) == "../")
- + ]
1230 [ # # # # ]: 0 : i = i.substr(3);
1231 [ + - - + : 17666 : else if(i.substr(0,2) == "./")
- + ]
1232 [ # # # # ]: 0 : i = i.substr(2);
1233 : :
1234 : : // 5.2.4 (2) B
1235 [ + - - + : 17666 : else if (i.substr(0,3) == "/./")
+ + ]
1236 [ + - + + ]: 281 : i = i.substr(2);
1237 [ - + ]: 8660 : else if (i == "/.")
1238 [ # # ]: 0 : i = ""; // no need to handle "/." complete-path-segment case; we're dealing with file names
1239 : :
1240 : : // 5.2.4 (2) C
1241 [ + - - + : 17320 : else if (i.substr(0,4) == "/../") {
+ + ]
1242 [ + - + + ]: 251 : i = i.substr(3);
1243 : 251 : string::size_type sl = o.rfind("/");
1244 [ + - ]: 251 : if (sl != string::npos)
1245 [ + - + - ]: 502 : o = o.substr(0, sl);
1246 : : else
1247 [ # # ]: 0 : o = "";
1248 [ - + ]: 8409 : } else if (i == "/..")
1249 [ # # ]: 0 : i = ""; // no need to handle "/.." complete-path-segment case; we're dealing with file names
1250 : :
1251 : : // 5.2.4 (2) D
1252 : : // no need to handle these cases; we're dealing with file names
1253 [ - + ]: 8409 : else if (i == ".")
1254 [ # # ]: 0 : i = "";
1255 [ - + ]: 8409 : else if (i == "..")
1256 [ # # ]: 0 : i = "";
1257 : :
1258 : : // POSIX special: map // to /
1259 [ + - - + : 16818 : else if (i.substr(0,2) == "//")
+ + ]
1260 [ + - + + ]: 64 : i = i.substr(1);
1261 : :
1262 : : // 5.2.4 (2) E
1263 : : else {
1264 [ - + ]: 8353 : string::size_type next_slash = i.find("/", (i[0]=='/' ? 1 : 0)); // skip first slash
1265 [ + - + + : 16706 : o += i.substr(0, next_slash);
- - ]
1266 [ + + ]: 8353 : if (next_slash == string::npos)
1267 [ + + + - ]: 12119 : i = "";
1268 : : else
1269 [ + - + + ]: 12050 : i = i.substr(next_slash);
1270 : : }
1271 : : }
1272 : :
1273 [ + - ]: 3286 : return o;
1274 : : }
1275 : :
1276 : :
1277 : : // Estimate available free space for a given filesystem via statfs(2).
1278 : : // Return true if the free fraction is known to be smaller than the
1279 : : // given minimum percentage. Also update a related metric.
1280 : 1685 : bool statfs_free_enough_p(const string& path, const string& label, long minfree = 0)
1281 : : {
1282 : 1685 : struct statfs sfs;
1283 : 1685 : int rc = statfs(path.c_str(), &sfs);
1284 [ + + ]: 1684 : if (rc == 0)
1285 : : {
1286 : 1659 : double s = (double) sfs.f_bavail / (double) sfs.f_blocks;
1287 [ + - + - : 4979 : set_metric("filesys_free_ratio","purpose",label, s);
+ - - - ]
1288 : 1660 : return ((s * 100.0) < minfree);
1289 : : }
1290 : : return false;
1291 : : }
1292 : :
1293 : :
1294 : :
1295 : : // A map-like class that owns a cache of file descriptors (indexed by
1296 : : // file / content names).
1297 : : //
1298 : : // If only it could use fd's instead of file names ... but we can't
1299 : : // dup(2) to create independent descriptors for the same unlinked
1300 : : // files, so would have to use some goofy linux /proc/self/fd/%d
1301 : : // hack such as the following
1302 : :
1303 : : #if 0
1304 : : int superdup(int fd)
1305 : : {
1306 : : #ifdef __linux__
1307 : : char *fdpath = NULL;
1308 : : int rc = asprintf(& fdpath, "/proc/self/fd/%d", fd);
1309 : : int newfd;
1310 : : if (rc >= 0)
1311 : : newfd = open(fdpath, O_RDONLY);
1312 : : else
1313 : : newfd = -1;
1314 : : free (fdpath);
1315 : : return newfd;
1316 : : #else
1317 : : return -1;
1318 : : #endif
1319 : : }
1320 : : #endif
1321 : :
1322 : : class libarchive_fdcache
1323 : : {
1324 : : private:
1325 : : mutex fdcache_lock;
1326 : :
1327 : : struct fdcache_entry
1328 : : {
1329 : : string archive;
1330 : : string entry;
1331 : : string fd;
1332 : : double fd_size_mb; // slightly rounded up megabytes
1333 : : };
1334 : : deque<fdcache_entry> lru; // @head: most recently used
1335 : : long max_fds;
1336 : : deque<fdcache_entry> prefetch; // prefetched
1337 : : long max_mbs;
1338 : : long max_prefetch_mbs;
1339 : : long max_prefetch_fds;
1340 : :
1341 : : public:
1342 : 1015 : void set_metrics()
1343 : : {
1344 : 1015 : double fdcache_mb = 0.0;
1345 : 1015 : double prefetch_mb = 0.0;
1346 [ + + + + ]: 4988 : for (auto i = lru.begin(); i < lru.end(); i++)
1347 : 1479 : fdcache_mb += i->fd_size_mb;
1348 [ + + + + ]: 3112 : for (auto j = prefetch.begin(); j < prefetch.end(); j++)
1349 : 541 : prefetch_mb += j->fd_size_mb;
1350 [ + - ]: 2030 : set_metric("fdcache_bytes", fdcache_mb*1024.0*1024.0);
1351 [ + - ]: 2030 : set_metric("fdcache_count", lru.size());
1352 [ + - ]: 2030 : set_metric("fdcache_prefetch_bytes", prefetch_mb*1024.0*1024.0);
1353 [ + - ]: 2030 : set_metric("fdcache_prefetch_count", prefetch.size());
1354 : 1015 : }
1355 : :
1356 : 586 : void intern(const string& a, const string& b, string fd, off_t sz, bool front_p)
1357 : : {
1358 : 586 : {
1359 : 586 : unique_lock<mutex> lock(fdcache_lock);
1360 : : // nuke preexisting copy
1361 [ + + + + ]: 2651 : for (auto i = lru.begin(); i < lru.end(); i++)
1362 : : {
1363 [ + + - + ]: 493 : if (i->archive == a && i->entry == b)
1364 : : {
1365 : 0 : unlink (i->fd.c_str());
1366 : 0 : lru.erase(i);
1367 [ # # # # : 0 : inc_metric("fdcache_op_count","op","dequeue");
# # # # #
# # # # #
# # ]
1368 : 0 : break; // must not continue iterating
1369 : : }
1370 : : }
1371 : : // nuke preexisting copy in prefetch
1372 [ + + + + ]: 1982 : for (auto i = prefetch.begin(); i < prefetch.end(); i++)
1373 : : {
1374 [ + + + - ]: 270 : if (i->archive == a && i->entry == b)
1375 : : {
1376 : 0 : unlink (i->fd.c_str());
1377 : 0 : prefetch.erase(i);
1378 [ # # # # : 0 : inc_metric("fdcache_op_count","op","prefetch_dequeue");
# # # # #
# # # # #
# # ]
1379 : 0 : break; // must not continue iterating
1380 : : }
1381 : : }
1382 : 586 : double mb = (sz+65535)/1048576.0; // round up to 64K block
1383 [ + - + - : 1172 : fdcache_entry n = { a, b, fd, mb };
+ - + - ]
1384 [ + + ]: 586 : if (front_p)
1385 : : {
1386 [ + - + - : 945 : inc_metric("fdcache_op_count","op","enqueue");
+ - + - -
+ + - + -
- - - - ]
1387 [ + - ]: 315 : lru.push_front(n);
1388 : : }
1389 : : else
1390 : : {
1391 [ + - + - : 813 : inc_metric("fdcache_op_count","op","prefetch_enqueue");
+ - + - -
+ + - + -
- - - - ]
1392 [ + - ]: 271 : prefetch.push_front(n);
1393 : : }
1394 [ + + ]: 586 : if (verbose > 3)
1395 [ + - + - ]: 1713 : obatched(clog) << "fdcache interned a=" << a << " b=" << b
1396 [ + - + - : 571 : << " fd=" << fd << " mb=" << mb << " front=" << front_p << endl;
+ - + - +
- + - + -
+ - + - ]
1397 : :
1398 [ + - ]: 586 : set_metrics();
1399 : : }
1400 : :
1401 : : // NB: we age the cache at lookup time too
1402 [ + - - + : 586 : if (statfs_free_enough_p(tmpdir, "tmpdir", fdcache_mintmp))
- + ]
1403 : : {
1404 [ # # # # : 0 : inc_metric("fdcache_op_count","op","emerg-flush");
# # # # #
# # # #
# ]
1405 [ # # # # ]: 0 : obatched(clog) << "fdcache emergency flush for filling tmpdir" << endl;
1406 : 0 : this->limit(0, 0, 0, 0); // emergency flush
1407 : : }
1408 [ + + ]: 586 : else if (front_p)
1409 : 315 : this->limit(max_fds, max_mbs, max_prefetch_fds, max_prefetch_mbs); // age cache if required
1410 : 586 : }
1411 : :
1412 : 337 : int lookup(const string& a, const string& b)
1413 : : {
1414 : 337 : int fd = -1;
1415 : 337 : {
1416 : 337 : unique_lock<mutex> lock(fdcache_lock);
1417 [ + + + + ]: 2162 : for (auto i = lru.begin(); i < lru.end(); i++)
1418 : : {
1419 [ + + + + ]: 516 : if (i->archive == a && i->entry == b)
1420 : : { // found it; move it to head of lru
1421 [ + - ]: 20 : fdcache_entry n = *i;
1422 : 20 : lru.erase(i); // invalidates i, so no more iteration!
1423 [ + - ]: 20 : lru.push_front(n);
1424 [ + - + - : 60 : inc_metric("fdcache_op_count","op","requeue_front");
+ - + - -
+ + - + -
- - - - ]
1425 [ + - ]: 20 : fd = open(n.fd.c_str(), O_RDONLY);
1426 : 20 : break;
1427 : : }
1428 : : }
1429 : : // Iterate through prefetch while fd == -1 to ensure that no duplication between lru and
1430 : : // prefetch occurs.
1431 [ + + + + : 1176 : for ( auto i = prefetch.begin(); fd == -1 && i < prefetch.end(); ++i)
+ + ]
1432 : : {
1433 [ + + + + ]: 263 : if (i->archive == a && i->entry == b)
1434 : : { // found it; take the entry from the prefetch deque to the lru deque, since it has now been accessed.
1435 [ + - ]: 2 : fdcache_entry n = *i;
1436 : 2 : prefetch.erase(i);
1437 [ + - ]: 2 : lru.push_front(n);
1438 [ + - + - : 6 : inc_metric("fdcache_op_count","op","prefetch_access");
+ - + - -
+ + - + -
- - - - ]
1439 [ + - ]: 2 : fd = open(n.fd.c_str(), O_RDONLY);
1440 : 2 : break;
1441 : : }
1442 : : }
1443 : : }
1444 : :
1445 [ + - - + : 337 : if (statfs_free_enough_p(tmpdir, "tmpdir", fdcache_mintmp))
- + ]
1446 : : {
1447 [ # # # # : 0 : inc_metric("fdcache_op_count","op","emerg-flush");
# # # # #
# # # #
# ]
1448 [ # # # # ]: 0 : obatched(clog) << "fdcache emergency flush for filling tmpdir" << endl;
1449 : 0 : this->limit(0, 0, 0, 0); // emergency flush
1450 : : }
1451 [ + + ]: 337 : else if (fd >= 0)
1452 : 22 : this->limit(max_fds, max_mbs, max_prefetch_fds, max_prefetch_mbs); // age cache if required
1453 : :
1454 : 337 : return fd;
1455 : : }
1456 : :
1457 : 596 : int probe(const string& a, const string& b) // just a cache residency check - don't modify LRU state, don't open
1458 : : {
1459 : 1192 : unique_lock<mutex> lock(fdcache_lock);
1460 [ + + + + ]: 2755 : for (auto i = lru.begin(); i < lru.end(); i++)
1461 : : {
1462 [ + + + + ]: 531 : if (i->archive == a && i->entry == b)
1463 : : {
1464 [ + - + - : 30 : inc_metric("fdcache_op_count","op","probe_hit");
+ - + - -
+ + - - -
- - - - ]
1465 : 10 : return true;
1466 : : }
1467 : : }
1468 [ + + + + ]: 1982 : for (auto i = prefetch.begin(); i < prefetch.end(); i++)
1469 : : {
1470 [ + + + - ]: 270 : if (i->archive == a && i->entry == b)
1471 : : {
1472 [ # # # # : 0 : inc_metric("fdcache_op_count","op","prefetch_probe_hit");
# # # # #
# # # # #
# # ]
1473 : 0 : return true;
1474 : : }
1475 : : }
1476 [ + - + - : 1758 : inc_metric("fdcache_op_count","op","probe_miss");
+ - + - -
+ + - - -
- - - - ]
1477 : 586 : return false;
1478 : : }
1479 : :
1480 : 0 : void clear(const string& a, const string& b)
1481 : : {
1482 : 0 : unique_lock<mutex> lock(fdcache_lock);
1483 [ # # # # ]: 0 : for (auto i = lru.begin(); i < lru.end(); i++)
1484 : : {
1485 [ # # # # ]: 0 : if (i->archive == a && i->entry == b)
1486 : : { // found it; erase it from lru
1487 [ # # ]: 0 : fdcache_entry n = *i;
1488 : 0 : lru.erase(i); // invalidates i, so no more iteration!
1489 [ # # # # : 0 : inc_metric("fdcache_op_count","op","clear");
# # # # #
# # # # #
# # ]
1490 : 0 : unlink (n.fd.c_str());
1491 [ # # ]: 0 : set_metrics();
1492 : 0 : return;
1493 : : }
1494 : : }
1495 [ # # # # ]: 0 : for (auto i = prefetch.begin(); i < prefetch.end(); i++)
1496 : : {
1497 [ # # # # ]: 0 : if (i->archive == a && i->entry == b)
1498 : : { // found it; erase it from lru
1499 [ # # ]: 0 : fdcache_entry n = *i;
1500 : 0 : prefetch.erase(i); // invalidates i, so no more iteration!
1501 [ # # # # : 0 : inc_metric("fdcache_op_count","op","prefetch_clear");
# # # # #
# # # # #
# # ]
1502 : 0 : unlink (n.fd.c_str());
1503 [ # # ]: 0 : set_metrics();
1504 : 0 : return;
1505 : : }
1506 : : }
1507 : : }
1508 : :
1509 : 461 : void limit(long maxfds, long maxmbs, long maxprefetchfds, long maxprefetchmbs , bool metrics_p = true)
1510 : : {
1511 [ + + + + : 461 : if (verbose > 3 && (this->max_fds != maxfds || this->max_mbs != maxmbs))
+ + ]
1512 [ + - + - : 152 : obatched(clog) << "fdcache limited to maxfds=" << maxfds << " maxmbs=" << maxmbs << endl;
+ - + - ]
1513 : :
1514 : 461 : unique_lock<mutex> lock(fdcache_lock);
1515 : 461 : this->max_fds = maxfds;
1516 : 461 : this->max_mbs = maxmbs;
1517 : 461 : this->max_prefetch_fds = maxprefetchfds;
1518 : 461 : this->max_prefetch_mbs = maxprefetchmbs;
1519 : 461 : long total_fd = 0;
1520 : 461 : double total_mb = 0.0;
1521 [ + + + + ]: 2935 : for (auto i = lru.begin(); i < lru.end(); i++)
1522 : : {
1523 : : // accumulate totals from most recently used one going backward
1524 : 937 : total_fd ++;
1525 [ + + ]: 937 : total_mb += i->fd_size_mb;
1526 [ + + - + ]: 937 : if (total_fd > this->max_fds || total_mb > this->max_mbs)
1527 : : {
1528 : : // found the cut here point!
1529 : :
1530 [ + + + + ]: 1483 : for (auto j = i; j < lru.end(); j++) // close all the fds from here on in
1531 : : {
1532 [ + + ]: 317 : if (verbose > 3)
1533 [ + - + - : 1244 : obatched(clog) << "fdcache evicted a=" << j->archive << " b=" << j->entry
+ - ]
1534 [ + - + - : 311 : << " fd=" << j->fd << " mb=" << j->fd_size_mb << endl;
+ - + - +
- + - +
- ]
1535 [ + + ]: 317 : if (metrics_p)
1536 [ + - + - : 834 : inc_metric("fdcache_op_count","op","evict");
+ - + - -
+ + - - -
- - - - ]
1537 : 317 : unlink (j->fd.c_str());
1538 : : }
1539 : :
1540 : 266 : lru.erase(i, lru.end()); // erase the nodes generally
1541 : 266 : break;
1542 : : }
1543 : : }
1544 : 461 : total_fd = 0;
1545 : 461 : total_mb = 0.0;
1546 [ + + + + ]: 922 : for(auto i = prefetch.begin(); i < prefetch.end(); i++){
1547 : : // accumulate totals from most recently used one going backward
1548 : 263 : total_fd ++;
1549 [ - + ]: 263 : total_mb += i->fd_size_mb;
1550 [ - + - - ]: 263 : if (total_fd > this->max_prefetch_fds || total_mb > this->max_prefetch_mbs)
1551 : : {
1552 : : // found the cut here point!
1553 [ + + + + ]: 1333 : for (auto j = i; j < prefetch.end(); j++) // close all the fds from here on in
1554 : : {
1555 [ + + ]: 269 : if (verbose > 3)
1556 [ + - + - : 1040 : obatched(clog) << "fdcache evicted from prefetch a=" << j->archive << " b=" << j->entry
+ - ]
1557 [ + - + - : 260 : << " fd=" << j->fd << " mb=" << j->fd_size_mb << endl;
+ - + - +
- + - +
- ]
1558 [ + + ]: 269 : if (metrics_p)
1559 [ + - + - : 792 : inc_metric("fdcache_op_count","op","prefetch_evict");
+ - + - -
+ + - - -
- - - - ]
1560 : 269 : unlink (j->fd.c_str());
1561 : : }
1562 : :
1563 : 263 : prefetch.erase(i, prefetch.end()); // erase the nodes generally
1564 : 263 : break;
1565 : : }
1566 : : }
1567 [ + + + - ]: 461 : if (metrics_p) set_metrics();
1568 : 461 : }
1569 : :
1570 : :
1571 : 32 : ~libarchive_fdcache()
1572 : 32 : {
1573 : : // unlink any fdcache entries in $TMPDIR
1574 : : // don't update metrics; those globals may be already destroyed
1575 : 32 : limit(0, 0, 0, 0, false);
1576 : 32 : }
1577 : : };
1578 : : static libarchive_fdcache fdcache;
1579 : :
1580 : :
1581 : : // For security/portability reasons, many distro-package archives have
1582 : : // a "./" in front of path names; others have nothing, others have
1583 : : // "/". Canonicalize them all to a single leading "/", with the
1584 : : // assumption that this matches the dwarf-derived file names too.
1585 : 1644 : string canonicalized_archive_entry_pathname(struct archive_entry *e)
1586 : : {
1587 : 3288 : string fn = archive_entry_pathname(e);
1588 [ - + ]: 1644 : if (fn.size() == 0)
1589 [ # # ]: 0 : return fn;
1590 [ - + ]: 1644 : if (fn[0] == '/')
1591 [ - - + + ]: 1644 : return fn;
1592 [ + + ]: 1644 : if (fn[0] == '.')
1593 [ + - ]: 1066 : return fn.substr(1);
1594 : : else
1595 [ + - + - : 1156 : return string("/")+fn;
- - ]
1596 : : }
1597 : :
1598 : :
1599 : :
1600 : : static struct MHD_Response*
1601 : 366 : handle_buildid_r_match (bool internal_req_p,
1602 : : int64_t b_mtime,
1603 : : const string& b_source0,
1604 : : const string& b_source1,
1605 : : int *result_fd)
1606 : : {
1607 : 366 : struct stat fs;
1608 : 366 : int rc = stat (b_source0.c_str(), &fs);
1609 [ + + ]: 366 : if (rc != 0)
1610 [ + - + - : 58 : throw libc_exception (errno, string("stat ") + b_source0);
+ - - + -
- ]
1611 : :
1612 [ - + ]: 337 : if ((int64_t) fs.st_mtime != b_mtime)
1613 : : {
1614 [ # # ]: 0 : if (verbose)
1615 [ # # # # ]: 0 : obatched(clog) << "mtime mismatch for " << b_source0 << endl;
1616 : 0 : return 0;
1617 : : }
1618 : :
1619 : : // check for a match in the fdcache first
1620 : 337 : int fd = fdcache.lookup(b_source0, b_source1);
1621 [ + + ]: 337 : while (fd >= 0) // got one!; NB: this is really an if() with a possible branch out to the end
1622 : : {
1623 : 22 : rc = fstat(fd, &fs);
1624 [ - + ]: 22 : if (rc < 0) // disappeared?
1625 : : {
1626 [ # # ]: 0 : if (verbose)
1627 [ # # # # ]: 0 : obatched(clog) << "cannot fstat fdcache " << b_source0 << endl;
1628 : 0 : close(fd);
1629 : 0 : fdcache.clear(b_source0, b_source1);
1630 : : break; // branch out of if "loop", to try new libarchive fetch attempt
1631 : : }
1632 : :
1633 : 22 : struct MHD_Response* r = MHD_create_response_from_fd (fs.st_size, fd);
1634 [ - + ]: 22 : if (r == 0)
1635 : : {
1636 [ # # ]: 0 : if (verbose)
1637 [ # # # # ]: 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
1638 : 0 : close(fd);
1639 : : break; // branch out of if "loop", to try new libarchive fetch attempt
1640 : : }
1641 : :
1642 [ + - + - : 66 : inc_metric ("http_responses_total","result","archive fdcache");
+ - - + +
- - - -
- ]
1643 : :
1644 : 22 : add_mhd_response_header (r, "Content-Type", "application/octet-stream");
1645 [ + - ]: 22 : add_mhd_response_header (r, "X-DEBUGINFOD-SIZE",
1646 : 22 : to_string(fs.st_size).c_str());
1647 : 22 : add_mhd_response_header (r, "X-DEBUGINFOD-ARCHIVE", b_source0.c_str());
1648 : 22 : add_mhd_response_header (r, "X-DEBUGINFOD-FILE", b_source1.c_str());
1649 : 22 : add_mhd_last_modified (r, fs.st_mtime);
1650 [ + - ]: 22 : if (verbose > 1)
1651 [ + - + - : 44 : obatched(clog) << "serving fdcache archive " << b_source0 << " file " << b_source1 << endl;
+ - + - ]
1652 : : /* libmicrohttpd will close it. */
1653 [ + - ]: 22 : if (result_fd)
1654 : 22 : *result_fd = fd;
1655 : : return r;
1656 : : // NB: see, we never go around the 'loop' more than once
1657 : : }
1658 : :
1659 : : // no match ... grumble, must process the archive
1660 : 652 : string archive_decoder = "/dev/null";
1661 [ + - + + ]: 630 : string archive_extension = "";
1662 [ + + ]: 901 : for (auto&& arch : scan_archives)
1663 [ + + ]: 586 : if (string_endswith(b_source0, arch.first))
1664 : : {
1665 [ + - ]: 315 : archive_extension = arch.first;
1666 [ + - ]: 901 : archive_decoder = arch.second;
1667 : : }
1668 : 315 : FILE* fp;
1669 : 315 : defer_dtor<FILE*,int>::dtor_fn dfn;
1670 [ + + ]: 315 : if (archive_decoder != "cat")
1671 : : {
1672 [ + - + - : 789 : string popen_cmd = archive_decoder + " " + shell_escape(b_source0);
+ - + + -
- - - ]
1673 [ + - ]: 263 : fp = popen (popen_cmd.c_str(), "r"); // "e" O_CLOEXEC?
1674 : 263 : dfn = pclose;
1675 [ - + ]: 263 : if (fp == NULL)
1676 [ # # # # : 0 : throw libc_exception (errno, string("popen ") + popen_cmd);
# # # # #
# ]
1677 : : }
1678 : : else
1679 : : {
1680 [ + - ]: 52 : fp = fopen (b_source0.c_str(), "r");
1681 : 52 : dfn = fclose;
1682 [ - + ]: 52 : if (fp == NULL)
1683 [ # # # # : 0 : throw libc_exception (errno, string("fopen ") + b_source0);
# # # # #
# ]
1684 : : }
1685 [ - + ]: 315 : defer_dtor<FILE*,int> fp_closer (fp, dfn);
1686 : :
1687 : 315 : struct archive *a;
1688 [ + - ]: 315 : a = archive_read_new();
1689 [ - + ]: 315 : if (a == NULL)
1690 [ # # # # ]: 0 : throw archive_exception("cannot create archive reader");
1691 : 315 : defer_dtor<struct archive*,int> archive_closer (a, archive_read_free);
1692 : :
1693 [ + - ]: 315 : rc = archive_read_support_format_all(a);
1694 [ - + ]: 315 : if (rc != ARCHIVE_OK)
1695 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all format");
1696 [ + - ]: 315 : rc = archive_read_support_filter_all(a);
1697 [ - + ]: 315 : if (rc != ARCHIVE_OK)
1698 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all filters");
1699 : :
1700 [ + - ]: 315 : rc = archive_read_open_FILE (a, fp);
1701 [ - + ]: 315 : if (rc != ARCHIVE_OK)
1702 [ # # # # ]: 0 : throw archive_exception(a, "cannot open archive from pipe");
1703 : :
1704 : : // archive traversal is in three stages, no, four stages:
1705 : : // 1) skip entries whose names do not match the requested one
1706 : : // 2) extract the matching entry name (set r = result)
1707 : : // 3) extract some number of prefetched entries (just into fdcache)
1708 : : // 4) abort any further processing
1709 : 315 : struct MHD_Response* r = 0; // will set in stage 2
1710 [ + + ]: 315 : unsigned prefetch_count =
1711 : : internal_req_p ? 0 : fdcache_prefetch; // will decrement in stage 3
1712 : :
1713 [ + + ]: 4954 : while(r == 0 || prefetch_count > 0) // stage 1, 2, or 3
1714 : : {
1715 [ + - ]: 4946 : if (interrupted)
1716 : : break;
1717 : :
1718 : 4946 : struct archive_entry *e;
1719 [ + - ]: 4946 : rc = archive_read_next_header (a, &e);
1720 [ + + ]: 4946 : if (rc != ARCHIVE_OK)
1721 : : break;
1722 : :
1723 [ + - + + ]: 4639 : if (! S_ISREG(archive_entry_mode (e))) // skip non-files completely
1724 : 3248 : continue;
1725 : :
1726 [ + - ]: 1391 : string fn = canonicalized_archive_entry_pathname (e);
1727 [ + + + + ]: 1391 : if ((r == 0) && (fn != b_source1)) // stage 1
1728 : 5434 : continue;
1729 : :
1730 [ + - + + ]: 596 : if (fdcache.probe (b_source0, fn)) // skip if already interned
1731 : 10 : continue;
1732 : :
1733 : : // extract this file to a temporary file
1734 : 586 : char* tmppath = NULL;
1735 : 586 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
1736 [ - + ]: 586 : if (rc < 0)
1737 [ # # # # ]: 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
1738 : 586 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
1739 [ + - ]: 586 : fd = mkstemp (tmppath);
1740 [ - + ]: 586 : if (fd < 0)
1741 [ # # # # ]: 0 : throw libc_exception (errno, "cannot create temporary file");
1742 : : // NB: don't unlink (tmppath), as fdcache will take charge of it.
1743 : :
1744 : : // NB: this can take many uninterruptible seconds for a huge file
1745 [ + - ]: 586 : rc = archive_read_data_into_fd (a, fd);
1746 [ - + ]: 586 : if (rc != ARCHIVE_OK) // e.g. ENOSPC!
1747 : : {
1748 [ # # ]: 0 : close (fd);
1749 : 0 : unlink (tmppath);
1750 [ # # # # ]: 0 : throw archive_exception(a, "cannot extract file");
1751 : : }
1752 : :
1753 : : // Set the mtime so the fdcache file mtimes, even prefetched ones,
1754 : : // propagate to future webapi clients.
1755 : 586 : struct timeval tvs[2];
1756 [ + - ]: 586 : tvs[0].tv_sec = tvs[1].tv_sec = archive_entry_mtime(e);
1757 : 586 : tvs[0].tv_usec = tvs[1].tv_usec = 0;
1758 : 586 : (void) futimes (fd, tvs); /* best effort */
1759 : :
1760 [ + + ]: 586 : if (r != 0) // stage 3
1761 : : {
1762 : : // NB: now we know we have a complete reusable file; make fdcache
1763 : : // responsible for unlinking it later.
1764 [ + - + - ]: 271 : fdcache.intern(b_source0, fn,
1765 : : tmppath, archive_entry_size(e),
1766 [ + - + - ]: 542 : false); // prefetched ones go to the prefetch cache
1767 : 271 : prefetch_count --;
1768 [ + - ]: 271 : close (fd); // we're not saving this fd to make a mhd-response from!
1769 [ + + ]: 1662 : continue;
1770 : : }
1771 : :
1772 : : // NB: now we know we have a complete reusable file; make fdcache
1773 : : // responsible for unlinking it later.
1774 [ + - + - ]: 315 : fdcache.intern(b_source0, b_source1,
1775 : : tmppath, archive_entry_size(e),
1776 [ + - + - ]: 630 : true); // requested ones go to the front of lru
1777 : :
1778 [ + - + - : 945 : inc_metric ("http_responses_total","result",archive_extension + " archive");
+ - + - -
+ + - + -
- - - - ]
1779 [ + - + - ]: 315 : r = MHD_create_response_from_fd (archive_entry_size(e), fd);
1780 [ - + ]: 315 : if (r == 0)
1781 : : {
1782 [ # # ]: 0 : if (verbose)
1783 [ # # # # : 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
# # ]
1784 [ # # ]: 0 : close(fd);
1785 [ # # ]: 0 : break; // assume no chance of better luck around another iteration; no other copies of same file
1786 : : }
1787 : : else
1788 : : {
1789 [ + - ]: 630 : std::string file = b_source1.substr(b_source1.find_last_of("/")+1, b_source1.length());
1790 [ + - ]: 315 : add_mhd_response_header (r, "Content-Type",
1791 : : "application/octet-stream");
1792 [ + - ]: 315 : add_mhd_response_header (r, "X-DEBUGINFOD-SIZE",
1793 [ + - + - ]: 315 : to_string(archive_entry_size(e)).c_str());
1794 [ + - ]: 315 : add_mhd_response_header (r, "X-DEBUGINFOD-ARCHIVE",
1795 : : b_source0.c_str());
1796 [ + - ]: 315 : add_mhd_response_header (r, "X-DEBUGINFOD-FILE", file.c_str());
1797 [ + - + - ]: 315 : add_mhd_last_modified (r, archive_entry_mtime(e));
1798 [ + - ]: 315 : if (verbose > 1)
1799 [ + - + - : 630 : obatched(clog) << "serving archive " << b_source0 << " file " << b_source1 << endl;
+ - + - +
- - - ]
1800 : : /* libmicrohttpd will close it. */
1801 [ + - ]: 315 : if (result_fd)
1802 : 315 : *result_fd = fd;
1803 [ + + ]: 315 : continue;
1804 : : }
1805 : : }
1806 : :
1807 : : // XXX: rpm/file not found: delete this R entry?
1808 : 315 : return r;
1809 : : }
1810 : :
1811 : :
1812 : : static struct MHD_Response*
1813 : 398 : handle_buildid_match (bool internal_req_p,
1814 : : int64_t b_mtime,
1815 : : const string& b_stype,
1816 : : const string& b_source0,
1817 : : const string& b_source1,
1818 : : int *result_fd)
1819 : : {
1820 : 398 : try
1821 : : {
1822 [ + + ]: 398 : if (b_stype == "F")
1823 [ + - ]: 32 : return handle_buildid_f_match(internal_req_p, b_mtime, b_source0, result_fd);
1824 [ + - ]: 366 : else if (b_stype == "R")
1825 [ + + ]: 366 : return handle_buildid_r_match(internal_req_p, b_mtime, b_source0, b_source1, result_fd);
1826 : : }
1827 [ - + ]: 58 : catch (const reportable_exception &e)
1828 : : {
1829 [ + - ]: 29 : e.report(clog);
1830 : : // Report but swallow libc etc. errors here; let the caller
1831 : : // iterate to other matches of the content.
1832 : : }
1833 : :
1834 : : return 0;
1835 : : }
1836 : :
1837 : :
1838 : : static int
1839 : 348 : debuginfod_find_progress (debuginfod_client *, long a, long b)
1840 : : {
1841 [ - + ]: 348 : if (verbose > 4)
1842 [ # # # # : 0 : obatched(clog) << "federated debuginfod progress=" << a << "/" << b << endl;
# # # # #
# ]
1843 : :
1844 : 348 : return interrupted;
1845 : : }
1846 : :
1847 : :
1848 : : // a little lru pool of debuginfod_client*s for reuse between query threads
1849 : :
1850 : : mutex dc_pool_lock;
1851 : : deque<debuginfod_client*> dc_pool;
1852 : :
1853 : 238 : debuginfod_client* debuginfod_pool_begin()
1854 : : {
1855 : 476 : unique_lock<mutex> lock(dc_pool_lock);
1856 [ + + ]: 238 : if (dc_pool.size() > 0)
1857 : : {
1858 [ + - + - : 672 : inc_metric("dc_pool_op_count","op","begin-reuse");
+ - + - -
+ + - - -
- - - - ]
1859 : 224 : debuginfod_client *c = dc_pool.front();
1860 : 224 : dc_pool.pop_front();
1861 : 224 : return c;
1862 : : }
1863 [ + - + - : 42 : inc_metric("dc_pool_op_count","op","begin-new");
+ - + - -
+ + - + -
- - - - -
- ]
1864 [ + - ]: 14 : return debuginfod_begin();
1865 : : }
1866 : :
1867 : :
1868 : 62 : void debuginfod_pool_groom()
1869 : : {
1870 : 62 : unique_lock<mutex> lock(dc_pool_lock);
1871 [ + + ]: 74 : while (dc_pool.size() > 0)
1872 : : {
1873 [ + - + - : 36 : inc_metric("dc_pool_op_count","op","end");
+ - + - -
+ + - + -
- - - - -
- ]
1874 [ + - ]: 12 : debuginfod_end(dc_pool.front());
1875 : 12 : dc_pool.pop_front();
1876 : : }
1877 : 62 : }
1878 : :
1879 : :
1880 : 236 : void debuginfod_pool_end(debuginfod_client* c)
1881 : : {
1882 : 236 : unique_lock<mutex> lock(dc_pool_lock);
1883 [ + - + - : 708 : inc_metric("dc_pool_op_count","op","end-save");
+ - + - -
+ + - + -
- - - - -
- ]
1884 [ + - ]: 236 : dc_pool.push_front(c); // accelerate reuse, vs. push_back
1885 : 236 : }
1886 : :
1887 : :
1888 : : static struct MHD_Response*
1889 : 610 : handle_buildid (MHD_Connection* conn,
1890 : : const string& buildid /* unsafe */,
1891 : : string& artifacttype /* unsafe, cleanse on exception/return */,
1892 : : const string& suffix /* unsafe */,
1893 : : int *result_fd)
1894 : : {
1895 : : // validate artifacttype
1896 : 981 : string atype_code;
1897 [ + + + - ]: 610 : if (artifacttype == "debuginfo") atype_code = "D";
1898 [ + + + - ]: 254 : else if (artifacttype == "executable") atype_code = "E";
1899 [ + + + - ]: 24 : else if (artifacttype == "source") atype_code = "S";
1900 : : else {
1901 [ + - ]: 2 : artifacttype = "invalid"; // PR28242 ensure http_resposes metrics don't propagate unclean user data
1902 [ + - + - ]: 6 : throw reportable_exception("invalid artifacttype");
1903 : : }
1904 : :
1905 [ + - + - : 2063 : inc_metric("http_requests_total", "type", artifacttype);
+ - + - -
- - + ]
1906 : :
1907 [ + + - + ]: 630 : if (atype_code == "S" && suffix == "")
1908 [ # # # # ]: 0 : throw reportable_exception("invalid source suffix");
1909 : :
1910 : : // validate buildid
1911 [ + + ]: 608 : if ((buildid.size() < 2) || // not empty
1912 [ + + + - : 1215 : (buildid.size() % 2) || // even number
+ - ]
1913 : 607 : (buildid.find_first_not_of("0123456789abcdef") != string::npos)) // pure tasty lowercase hex
1914 [ + - - + ]: 2 : throw reportable_exception("invalid buildid");
1915 : :
1916 [ + - ]: 607 : if (verbose > 1)
1917 [ + - + - ]: 1821 : obatched(clog) << "searching for buildid=" << buildid << " artifacttype=" << artifacttype
1918 [ + - + - : 607 : << " suffix=" << suffix << endl;
+ - + - +
- ]
1919 : :
1920 : : // If invoked from the scanner threads, use the scanners' read-write
1921 : : // connection. Otherwise use the web query threads' read-only connection.
1922 [ + + ]: 607 : sqlite3 *thisdb = (conn == 0) ? db : dbq;
1923 : :
1924 : 607 : sqlite_ps *pp = 0;
1925 : :
1926 [ + + ]: 607 : if (atype_code == "D")
1927 : : {
1928 [ + - + - ]: 712 : pp = new sqlite_ps (thisdb, "mhd-query-d",
1929 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_d where buildid = ? "
1930 [ + - + - : 712 : "order by mtime desc");
+ - + - -
+ + - -
- ]
1931 [ + - ]: 356 : pp->reset();
1932 [ + - ]: 356 : pp->bind(1, buildid);
1933 : : }
1934 [ + + ]: 251 : else if (atype_code == "E")
1935 : : {
1936 [ + - + - ]: 458 : pp = new sqlite_ps (thisdb, "mhd-query-e",
1937 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_e where buildid = ? "
1938 [ + - + - : 458 : "order by mtime desc");
+ - + - -
+ + - -
- ]
1939 [ + - ]: 229 : pp->reset();
1940 [ + - ]: 229 : pp->bind(1, buildid);
1941 : : }
1942 [ + - ]: 22 : else if (atype_code == "S")
1943 : : {
1944 : : // PR25548
1945 : : // Incoming source queries may come in with either dwarf-level OR canonicalized paths.
1946 : : // We let the query pass with either one.
1947 : :
1948 [ + - + - ]: 44 : pp = new sqlite_ps (thisdb, "mhd-query-s",
1949 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_s where buildid = ? and artifactsrc in (?,?) "
1950 [ + - + - : 44 : "order by sharedprefix(source0,source0ref) desc, mtime desc");
+ - + - -
+ + - -
- ]
1951 [ + - ]: 22 : pp->reset();
1952 [ + - ]: 22 : pp->bind(1, buildid);
1953 : : // NB: we don't store the non-canonicalized path names any more, but old databases
1954 : : // might have them (and no canon ones), so we keep searching for both.
1955 [ + - ]: 22 : pp->bind(2, suffix);
1956 [ + - + - : 283 : pp->bind(3, canon_pathname(suffix));
- + ]
1957 : : }
1958 [ - + ]: 978 : unique_ptr<sqlite_ps> ps_closer(pp); // release pp if exception or return
1959 : :
1960 : : // consume all the rows
1961 : 636 : while (1)
1962 : : {
1963 [ + - ]: 636 : int rc = pp->step();
1964 [ + + ]: 636 : if (rc == SQLITE_DONE) break;
1965 [ - + ]: 398 : if (rc != SQLITE_ROW)
1966 [ # # # # ]: 0 : throw sqlite_exception(rc, "step");
1967 : :
1968 [ + - ]: 398 : int64_t b_mtime = sqlite3_column_int64 (*pp, 0);
1969 [ + - - + : 427 : string b_stype = string((const char*) sqlite3_column_text (*pp, 1) ?: ""); /* by DDL may not be NULL */
+ - ]
1970 [ + - - + : 427 : string b_source0 = string((const char*) sqlite3_column_text (*pp, 2) ?: ""); /* may be NULL */
+ - - + ]
1971 [ + - + + : 459 : string b_source1 = string((const char*) sqlite3_column_text (*pp, 3) ?: ""); /* may be NULL */
+ - + - ]
1972 : :
1973 [ + - ]: 398 : if (verbose > 1)
1974 [ + - + - : 1194 : obatched(clog) << "found mtime=" << b_mtime << " stype=" << b_stype
- - ]
1975 [ + - + - : 398 : << " source0=" << b_source0 << " source1=" << b_source1 << endl;
+ - + - +
- + - +
- ]
1976 : :
1977 : : // Try accessing the located match.
1978 : : // XXX: in case of multiple matches, attempt them in parallel?
1979 [ + - ]: 398 : auto r = handle_buildid_match (conn ? false : true,
1980 : : b_mtime, b_stype, b_source0, b_source1, result_fd);
1981 [ + + ]: 398 : if (r)
1982 [ + + + - ]: 706 : return r;
1983 : : }
1984 [ + - ]: 238 : pp->reset();
1985 : :
1986 : : // We couldn't find it in the database. Last ditch effort
1987 : : // is to defer to other debuginfo servers.
1988 : :
1989 : 238 : int fd = -1;
1990 [ + - ]: 238 : debuginfod_client *client = debuginfod_pool_begin ();
1991 [ + - ]: 238 : if (client != NULL)
1992 : : {
1993 [ + - ]: 238 : debuginfod_set_progressfn (client, & debuginfod_find_progress);
1994 : :
1995 [ + - ]: 238 : if (conn)
1996 : : {
1997 : : // Transcribe incoming User-Agent:
1998 [ + - - + : 476 : string ua = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";
+ - ]
1999 [ + - + - : 714 : string ua_complete = string("User-Agent: ") + ua;
+ - + + +
- ]
2000 [ + - ]: 238 : debuginfod_add_http_header (client, ua_complete.c_str());
2001 : :
2002 : : // Compute larger XFF:, for avoiding info loss during
2003 : : // federation, and for future cyclicity detection.
2004 [ + - + + : 705 : string xff = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";
+ - + - ]
2005 [ + + ]: 238 : if (xff != "")
2006 [ + - - + : 20 : xff += string(", "); // comma separated list
- + ]
2007 : :
2008 : 238 : unsigned int xff_count = 0;
2009 [ + + ]: 358 : for (auto&& i : xff){
2010 [ + + ]: 120 : if (i == ',') xff_count++;
2011 : : }
2012 : :
2013 : : // if X-Forwarded-For: exceeds N hops,
2014 : : // do not delegate a local lookup miss to upstream debuginfods.
2015 [ + + ]: 238 : if (xff_count >= forwarded_ttl_limit)
2016 [ + - ]: 2 : throw reportable_exception(MHD_HTTP_NOT_FOUND, "not found, --forwared-ttl-limit reached \
2017 [ + - ]: 4 : and will not query the upstream servers");
2018 : :
2019 : : // Compute the client's numeric IP address only - so can't merge with conninfo()
2020 [ + - ]: 236 : const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
2021 : : MHD_CONNECTION_INFO_CLIENT_ADDRESS);
2022 [ + - ]: 236 : struct sockaddr *so = u ? u->client_addr : 0;
2023 : 236 : char hostname[256] = ""; // RFC1035
2024 [ + - - + ]: 236 : if (so && so->sa_family == AF_INET) {
2025 [ # # ]: 0 : (void) getnameinfo (so, sizeof (struct sockaddr_in), hostname, sizeof (hostname), NULL, 0,
2026 : : NI_NUMERICHOST);
2027 [ + - + - ]: 236 : } else if (so && so->sa_family == AF_INET6) {
2028 : 236 : struct sockaddr_in6* addr6 = (struct sockaddr_in6*) so;
2029 [ + - + - : 236 : if (IN6_IS_ADDR_V4MAPPED(&addr6->sin6_addr)) {
- + ]
2030 : 236 : struct sockaddr_in addr4;
2031 [ + - ]: 236 : memset (&addr4, 0, sizeof(addr4));
2032 : 236 : addr4.sin_family = AF_INET;
2033 : 236 : addr4.sin_port = addr6->sin6_port;
2034 [ + - ]: 236 : memcpy (&addr4.sin_addr.s_addr, addr6->sin6_addr.s6_addr+12, sizeof(addr4.sin_addr.s_addr));
2035 [ + - ]: 236 : (void) getnameinfo ((struct sockaddr*) &addr4, sizeof (addr4),
2036 : : hostname, sizeof (hostname), NULL, 0,
2037 : : NI_NUMERICHOST);
2038 : : } else {
2039 [ # # ]: 0 : (void) getnameinfo (so, sizeof (struct sockaddr_in6), hostname, sizeof (hostname), NULL, 0,
2040 : : NI_NUMERICHOST);
2041 : : }
2042 : : }
2043 : :
2044 [ + - + - : 710 : string xff_complete = string("X-Forwarded-For: ")+xff+string(hostname);
+ - + - -
+ - + + -
+ + - - -
- - + ]
2045 [ + - ]: 236 : debuginfod_add_http_header (client, xff_complete.c_str());
2046 : : }
2047 : :
2048 [ + + ]: 236 : if (artifacttype == "debuginfo")
2049 [ + - ]: 34 : fd = debuginfod_find_debuginfo (client,
2050 [ + - ]: 34 : (const unsigned char*) buildid.c_str(),
2051 : : 0, NULL);
2052 [ + + ]: 202 : else if (artifacttype == "executable")
2053 [ + - ]: 201 : fd = debuginfod_find_executable (client,
2054 [ + - ]: 201 : (const unsigned char*) buildid.c_str(),
2055 : : 0, NULL);
2056 [ + - ]: 1 : else if (artifacttype == "source")
2057 [ + - ]: 1 : fd = debuginfod_find_source (client,
2058 [ + - ]: 1 : (const unsigned char*) buildid.c_str(),
2059 : : 0, suffix.c_str(), NULL);
2060 : : }
2061 : : else
2062 : 0 : fd = -errno; /* Set by debuginfod_begin. */
2063 [ + - ]: 236 : debuginfod_pool_end (client);
2064 : :
2065 [ + + ]: 236 : if (fd >= 0)
2066 : : {
2067 [ + - + - : 6 : inc_metric ("http_responses_total","result","upstream");
+ - + - -
+ + - - -
- - ]
2068 : 2 : struct stat s;
2069 : 2 : int rc = fstat (fd, &s);
2070 [ + - ]: 2 : if (rc == 0)
2071 : : {
2072 [ + - ]: 2 : auto r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd);
2073 [ + - ]: 2 : if (r)
2074 : : {
2075 [ + - ]: 2 : add_mhd_response_header (r, "Content-Type",
2076 : : "application/octet-stream");
2077 [ + - ]: 2 : add_mhd_last_modified (r, s.st_mtime);
2078 [ + - ]: 2 : if (verbose > 1)
2079 [ + - + - ]: 240 : obatched(clog) << "serving file from upstream debuginfod/cache" << endl;
2080 [ + - ]: 2 : if (result_fd)
2081 : 2 : *result_fd = fd;
2082 : 2 : return r; // NB: don't close fd; libmicrohttpd will
2083 : : }
2084 : : }
2085 [ # # ]: 0 : close (fd);
2086 : : }
2087 : : else
2088 [ + + ]: 234 : switch(fd)
2089 : : {
2090 : : case -ENOSYS:
2091 : : break;
2092 : : case -ENOENT:
2093 : : break;
2094 : 110 : default: // some more tricky error
2095 [ + - + - ]: 220 : throw libc_exception(-fd, "upstream debuginfod query failed");
2096 : : }
2097 : :
2098 [ + - - + ]: 248 : throw reportable_exception(MHD_HTTP_NOT_FOUND, "not found");
2099 : : }
2100 : :
2101 : :
2102 : : ////////////////////////////////////////////////////////////////////////
2103 : :
2104 : : static map<string,double> metrics; // arbitrary data for /metrics query
2105 : : // NB: store int64_t since all our metrics are integers; prometheus accepts double
2106 : : static mutex metrics_lock;
2107 : : // NB: these objects get released during the process exit via global dtors
2108 : : // do not call them from within other global dtors
2109 : :
2110 : : // utility function for assembling prometheus-compatible
2111 : : // name="escaped-value" strings
2112 : : // https://prometheus.io/docs/instrumenting/exposition_formats/
2113 : : static string
2114 : 57427 : metric_label(const string& name, const string& value)
2115 : : {
2116 : 57427 : string x = name + "=\"";
2117 [ + + ]: 748938 : for (auto&& c : value)
2118 [ - - - + ]: 691526 : switch(c)
2119 : : {
2120 [ # # ]: 0 : case '\\': x += "\\\\"; break;
2121 [ # # ]: 0 : case '\"': x += "\\\""; break;
2122 [ # # ]: 0 : case '\n': x += "\\n"; break;
2123 [ + - ]: 1383062 : default: x += c; break;
2124 : : }
2125 [ + - ]: 57412 : x += "\"";
2126 : 57412 : return x;
2127 : : }
2128 : :
2129 : :
2130 : : // add prometheus-format metric name + label tuple (if any) + value
2131 : :
2132 : : static void
2133 : 4124 : set_metric(const string& metric, double value)
2134 : : {
2135 : 4124 : unique_lock<mutex> lock(metrics_lock);
2136 [ + - + - ]: 4124 : metrics[metric] = value;
2137 : 4124 : }
2138 : : #if 0 /* unused */
2139 : : static void
2140 : : inc_metric(const string& metric)
2141 : : {
2142 : : unique_lock<mutex> lock(metrics_lock);
2143 : : metrics[metric] ++;
2144 : : }
2145 : : #endif
2146 : : static void
2147 : 2950 : set_metric(const string& metric,
2148 : : const string& lname, const string& lvalue,
2149 : : double value)
2150 : : {
2151 [ + - + - : 7651 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
+ - + + -
+ - - -
- ]
2152 [ + - + - ]: 5902 : unique_lock<mutex> lock(metrics_lock);
2153 [ + - + - ]: 2951 : metrics[key] = value;
2154 : 2951 : }
2155 : :
2156 : : static void
2157 : 22739 : inc_metric(const string& metric,
2158 : : const string& lname, const string& lvalue)
2159 : : {
2160 [ + - + - : 63401 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
+ - + + +
+ - - -
- ]
2161 [ + - + - ]: 45476 : unique_lock<mutex> lock(metrics_lock);
2162 [ + - + # ]: 22739 : metrics[key] ++;
2163 : 22739 : }
2164 : : static void
2165 : 20887 : add_metric(const string& metric,
2166 : : const string& lname, const string& lvalue,
2167 : : double value)
2168 : : {
2169 [ + - + - : 61111 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
+ - + + +
+ - - -
- ]
2170 [ + - + - ]: 41794 : unique_lock<mutex> lock(metrics_lock);
2171 [ + - + - ]: 20898 : metrics[key] += value;
2172 : 20898 : }
2173 : : #if 0
2174 : : static void
2175 : : add_metric(const string& metric,
2176 : : double value)
2177 : : {
2178 : : unique_lock<mutex> lock(metrics_lock);
2179 : : metrics[metric] += value;
2180 : : }
2181 : : #endif
2182 : :
2183 : :
2184 : : // and more for higher arity labels if needed
2185 : :
2186 : : static void
2187 : 2712 : inc_metric(const string& metric,
2188 : : const string& lname, const string& lvalue,
2189 : : const string& rname, const string& rvalue)
2190 : : {
2191 : 2712 : string key = (metric + "{"
2192 [ + - + - : 5424 : + metric_label(lname, lvalue) + ","
+ - - + -
+ - + - -
- - - - ]
2193 [ + - + - : 7134 : + metric_label(rname, rvalue) + "}");
+ - + + -
+ - - -
- ]
2194 [ + - + - ]: 5424 : unique_lock<mutex> lock(metrics_lock);
2195 [ + - + - ]: 2712 : metrics[key] ++;
2196 : 2712 : }
2197 : : static void
2198 : 2712 : add_metric(const string& metric,
2199 : : const string& lname, const string& lvalue,
2200 : : const string& rname, const string& rvalue,
2201 : : double value)
2202 : : {
2203 : 2712 : string key = (metric + "{"
2204 [ + - + - : 5424 : + metric_label(lname, lvalue) + ","
+ - - + -
+ - + - -
- - - - ]
2205 [ + - + - : 7134 : + metric_label(rname, rvalue) + "}");
+ - + + -
+ - - -
- ]
2206 [ + - + - ]: 5424 : unique_lock<mutex> lock(metrics_lock);
2207 [ + - + - ]: 2712 : metrics[key] += value;
2208 : 2712 : }
2209 : :
2210 : : static struct MHD_Response*
2211 : 307 : handle_metrics (off_t* size)
2212 : : {
2213 : 307 : stringstream o;
2214 : 307 : {
2215 [ + - ]: 307 : unique_lock<mutex> lock(metrics_lock);
2216 [ + + ]: 26501 : for (auto&& i : metrics)
2217 [ + - ]: 26194 : o << i.first
2218 : : << " "
2219 [ + - + - ]: 26194 : << std::setprecision(std::numeric_limits<double>::digits10 + 1)
2220 [ + - + - ]: 26194 : << i.second
2221 : 26194 : << endl;
2222 : : }
2223 [ + - ]: 614 : const string& os = o.str();
2224 [ + - ]: 307 : MHD_Response* r = MHD_create_response_from_buffer (os.size(),
2225 [ + - ]: 307 : (void*) os.c_str(),
2226 : : MHD_RESPMEM_MUST_COPY);
2227 [ + - ]: 307 : if (r != NULL)
2228 : : {
2229 [ + - ]: 307 : *size = os.size();
2230 [ + - ]: 307 : add_mhd_response_header (r, "Content-Type", "text/plain");
2231 : : }
2232 [ + - ]: 614 : return r;
2233 : : }
2234 : :
2235 : : static struct MHD_Response*
2236 : 0 : handle_root (off_t* size)
2237 : : {
2238 [ # # # # : 0 : static string version = "debuginfod (" + string (PACKAGE_NAME) + ") "
# # # # #
# # # #
# ]
2239 [ # # # # : 0 : + string (PACKAGE_VERSION);
# # # # #
# # # # #
# # ]
2240 : 0 : MHD_Response* r = MHD_create_response_from_buffer (version.size (),
2241 : 0 : (void *) version.c_str (),
2242 : : MHD_RESPMEM_PERSISTENT);
2243 [ # # ]: 0 : if (r != NULL)
2244 : : {
2245 : 0 : *size = version.size ();
2246 : 0 : add_mhd_response_header (r, "Content-Type", "text/plain");
2247 : : }
2248 : 0 : return r;
2249 : : }
2250 : :
2251 : :
2252 : : ////////////////////////////////////////////////////////////////////////
2253 : :
2254 : :
2255 : : /* libmicrohttpd callback */
2256 : : static MHD_RESULT
2257 : 1806 : handler_cb (void * /*cls*/,
2258 : : struct MHD_Connection *connection,
2259 : : const char *url,
2260 : : const char *method,
2261 : : const char * /*version*/,
2262 : : const char * /*upload_data*/,
2263 : : size_t * /*upload_data_size*/,
2264 : : void ** ptr)
2265 : : {
2266 : 1806 : struct MHD_Response *r = NULL;
2267 : 3614 : string url_copy = url;
2268 : :
2269 : : /* libmicrohttpd always makes (at least) two callbacks: once just
2270 : : past the headers, and one after the request body is finished
2271 : : being received. If we process things early (first callback) and
2272 : : queue a response, libmicrohttpd would suppress http keep-alive
2273 : : (via connection->read_closed = true). */
2274 : 1807 : static int aptr; /* just some random object to use as a flag */
2275 [ + + ]: 1807 : if (&aptr != *ptr)
2276 : : {
2277 : : /* do never respond on first call */
2278 : 904 : *ptr = &aptr;
2279 : 904 : return MHD_YES;
2280 : : }
2281 : 903 : *ptr = NULL; /* reset when done */
2282 : :
2283 [ + - ]: 903 : const char *maxsize_string = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, "X-DEBUGINFOD-MAXSIZE");
2284 : 903 : long maxsize = 0;
2285 [ + + + - ]: 903 : if (maxsize_string != NULL && maxsize_string[0] != '\0')
2286 : 1 : maxsize = atol(maxsize_string);
2287 : : else
2288 : : maxsize = 0;
2289 : :
2290 : : #if MHD_VERSION >= 0x00097002
2291 : 903 : enum MHD_Result rc;
2292 : : #else
2293 : : int rc = MHD_NO; // mhd
2294 : : #endif
2295 : 903 : int http_code = 500;
2296 : 903 : off_t http_size = -1;
2297 : 903 : struct timespec ts_start, ts_end;
2298 : 903 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
2299 : 904 : double afteryou = 0.0;
2300 [ + - - + : 3616 : string artifacttype, suffix;
+ + ]
2301 : :
2302 : 904 : try
2303 : : {
2304 [ + - - + : 1807 : if (string(method) != "GET")
- + ]
2305 [ # # # # ]: 0 : throw reportable_exception(400, "we support GET only");
2306 : :
2307 : : /* Start decoding the URL. */
2308 : 903 : size_t slash1 = url_copy.find('/', 1);
2309 [ + - ]: 1808 : string url1 = url_copy.substr(0, slash1); // ok even if slash1 not found
2310 : :
2311 [ + + + - ]: 1496 : if (slash1 != string::npos && url1 == "/buildid")
2312 : : {
2313 : : // PR27863: block this thread awhile if another thread is already busy
2314 : : // fetching the exact same thing. This is better for Everyone.
2315 : : // The latecomer says "... after you!" and waits.
2316 [ + - + - : 1430 : add_metric ("thread_busy", "role", "http-buildid-after-you", 1);
+ - + - -
+ - + - -
- - - + ]
2317 : : #ifdef HAVE_PTHREAD_SETNAME_NP
2318 : 594 : (void) pthread_setname_np (pthread_self(), "mhd-buildid-after-you");
2319 : : #endif
2320 : 594 : struct timespec tsay_start, tsay_end;
2321 : 594 : clock_gettime (CLOCK_MONOTONIC, &tsay_start);
2322 [ + + + - ]: 594 : static unique_set<string> busy_urls;
2323 [ + - ]: 949 : unique_set_reserver<string> after_you(busy_urls, url_copy);
2324 : 594 : clock_gettime (CLOCK_MONOTONIC, &tsay_end);
2325 : 594 : afteryou = (tsay_end.tv_sec - tsay_start.tv_sec) + (tsay_end.tv_nsec - tsay_start.tv_nsec)/1.e9;
2326 [ + - + - : 1427 : add_metric ("thread_busy", "role", "http-buildid-after-you", -1);
+ - + - -
+ - + + -
- - - - ]
2327 : :
2328 [ + - + - : 1666 : tmp_inc_metric m ("thread_busy", "role", "http-buildid");
+ - + - -
+ - + - -
- - ]
2329 : : #ifdef HAVE_PTHREAD_SETNAME_NP
2330 : 594 : (void) pthread_setname_np (pthread_self(), "mhd-buildid");
2331 : : #endif
2332 : 594 : size_t slash2 = url_copy.find('/', slash1+1);
2333 [ - + ]: 594 : if (slash2 == string::npos)
2334 [ # # # # ]: 0 : throw reportable_exception("/buildid/ webapi error, need buildid");
2335 : :
2336 [ + - ]: 1188 : string buildid = url_copy.substr(slash1+1, slash2-slash1-1);
2337 : :
2338 : 594 : size_t slash3 = url_copy.find('/', slash2+1);
2339 : :
2340 [ + + ]: 594 : if (slash3 == string::npos)
2341 : : {
2342 [ + - - + ]: 572 : artifacttype = url_copy.substr(slash2+1);
2343 [ + - ]: 572 : suffix = "";
2344 : : }
2345 : : else
2346 : : {
2347 [ + - - + ]: 22 : artifacttype = url_copy.substr(slash2+1, slash3-slash2-1);
2348 [ + - - + ]: 22 : suffix = url_copy.substr(slash3); // include the slash in the suffix
2349 : : }
2350 : :
2351 : : // get the resulting fd so we can report its size
2352 : 594 : int fd;
2353 [ + + ]: 594 : r = handle_buildid(connection, buildid, artifacttype, suffix, &fd);
2354 [ + - ]: 355 : if (r)
2355 : : {
2356 : 355 : struct stat fs;
2357 [ + - ]: 355 : if (fstat(fd, &fs) == 0)
2358 : 355 : http_size = fs.st_size;
2359 : : // libmicrohttpd will close (fd);
2360 : : }
2361 : : }
2362 [ + + ]: 310 : else if (url1 == "/metrics")
2363 : : {
2364 [ + - + - : 614 : tmp_inc_metric m ("thread_busy", "role", "http-metrics");
+ - + - -
+ - + + -
- - - - ]
2365 [ + - ]: 307 : artifacttype = "metrics";
2366 [ + - + - : 921 : inc_metric("http_requests_total", "type", artifacttype);
+ - + - +
- - - ]
2367 [ + - ]: 307 : r = handle_metrics(& http_size);
2368 : : }
2369 [ - + ]: 3 : else if (url1 == "/")
2370 : : {
2371 [ # # ]: 0 : artifacttype = "/";
2372 [ - - - - : 243 : inc_metric("http_requests_total", "type", artifacttype);
- - - - -
- - - -
+ ]
2373 [ # # ]: 0 : r = handle_root(& http_size);
2374 : : }
2375 : : else
2376 [ + - + - : 9 : throw reportable_exception("webapi error, unrecognized '" + url1 + "'");
+ - - + -
- ]
2377 : :
2378 [ - + ]: 662 : if (r == 0)
2379 [ # # # # ]: 0 : throw reportable_exception("internal error, missing response");
2380 : :
2381 [ + + + - ]: 662 : if (maxsize > 0 && http_size > maxsize)
2382 : : {
2383 [ + - ]: 1 : MHD_destroy_response(r);
2384 [ + - + - : 3 : throw reportable_exception(406, "File too large, max size=" + std::to_string(maxsize));
+ - - + -
- ]
2385 : : }
2386 : :
2387 [ + - ]: 661 : rc = MHD_queue_response (connection, MHD_HTTP_OK, r);
2388 : 661 : http_code = MHD_HTTP_OK;
2389 [ + - ]: 661 : MHD_destroy_response (r);
2390 : : }
2391 [ - + ]: 243 : catch (const reportable_exception& e)
2392 : : {
2393 [ + - + - : 729 : inc_metric("http_responses_total","result","error");
+ - + - -
+ + - + -
- - - - ]
2394 [ + - ]: 243 : e.report(clog);
2395 : 243 : http_code = e.code;
2396 [ + - ]: 243 : http_size = e.message.size();
2397 [ + - ]: 243 : rc = e.mhd_send_response (connection);
2398 : : }
2399 : :
2400 : 904 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
2401 : 904 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
2402 : : // afteryou: delay waiting for other client's identical query to complete
2403 : : // deltas: total latency, including afteryou waiting
2404 [ + - + - : 2712 : obatched(clog) << conninfo(connection)
+ - - - ]
2405 : : << ' ' << method << ' ' << url
2406 [ + - + - : 1808 : << ' ' << http_code << ' ' << http_size
+ - + - +
- + - +
- ]
2407 [ + - + - : 1808 : << ' ' << (int)(afteryou*1000) << '+' << (int)((deltas-afteryou)*1000) << "ms"
+ - + - +
- + - ]
2408 [ + - ]: 904 : << endl;
2409 : :
2410 : : // related prometheus metrics
2411 [ + - + + ]: 1808 : string http_code_str = to_string(http_code);
2412 [ + - + - ]: 1808 : add_metric("http_responses_transfer_bytes_sum",
2413 [ + - + - : 2712 : "code", http_code_str, "type", artifacttype, http_size);
+ - - + +
- + - - -
- - - - ]
2414 [ + - + - ]: 1808 : inc_metric("http_responses_transfer_bytes_count",
2415 [ + - + - : 2712 : "code", http_code_str, "type", artifacttype);
+ - - + +
- + - - -
- - ]
2416 : :
2417 [ + - + - ]: 1808 : add_metric("http_responses_duration_milliseconds_sum",
2418 [ + - + - : 2712 : "code", http_code_str, "type", artifacttype, deltas*1000); // prometheus prefers _seconds and floating point
+ - - + +
- + - - -
- - ]
2419 [ + - + - ]: 1808 : inc_metric("http_responses_duration_milliseconds_count",
2420 [ + - + - : 2712 : "code", http_code_str, "type", artifacttype);
+ - - + +
- + - - -
- - ]
2421 : :
2422 [ + - + - ]: 1808 : add_metric("http_responses_after_you_milliseconds_sum",
2423 [ + - + - : 2712 : "code", http_code_str, "type", artifacttype, afteryou*1000);
+ - - + +
- + - - -
- - ]
2424 [ + - + - ]: 1808 : inc_metric("http_responses_after_you_milliseconds_count",
2425 [ + - + - : 2712 : "code", http_code_str, "type", artifacttype);
+ - - + +
- - + - -
- - - - ]
2426 : :
2427 [ - + ]: 904 : return rc;
2428 : : }
2429 : :
2430 : :
2431 : : ////////////////////////////////////////////////////////////////////////
2432 : : // borrowed originally from src/nm.c get_local_names()
2433 : :
2434 : : static void
2435 : 82 : dwarf_extract_source_paths (Elf *elf, set<string>& debug_sourcefiles)
2436 : : noexcept // no exceptions - so we can simplify the altdbg resource release at end
2437 : : {
2438 : 82 : Dwarf* dbg = dwarf_begin_elf (elf, DWARF_C_READ, NULL);
2439 [ - + ]: 82 : if (dbg == NULL)
2440 : 0 : return;
2441 : :
2442 : 82 : Dwarf* altdbg = NULL;
2443 : 82 : int altdbg_fd = -1;
2444 : :
2445 : : // DWZ handling: if we have an unsatisfied debug-alt-link, add an
2446 : : // empty string into the outgoing sourcefiles set, so the caller
2447 : : // should know that our data is incomplete.
2448 : 82 : const char *alt_name_p;
2449 : 82 : const void *alt_build_id; // elfutils-owned memory
2450 : 82 : ssize_t sz = dwelf_dwarf_gnu_debugaltlink (dbg, &alt_name_p, &alt_build_id);
2451 [ + + ]: 82 : if (sz > 0) // got one!
2452 : : {
2453 : 32 : string buildid;
2454 : 16 : unsigned char* build_id_bytes = (unsigned char*) alt_build_id;
2455 [ + + ]: 336 : for (ssize_t idx=0; idx<sz; idx++)
2456 : : {
2457 : 320 : buildid += "0123456789abcdef"[build_id_bytes[idx] >> 4];
2458 : 320 : buildid += "0123456789abcdef"[build_id_bytes[idx] & 0xf];
2459 : : }
2460 : :
2461 [ + - ]: 16 : if (verbose > 3)
2462 : 16 : obatched(clog) << "Need altdebug buildid=" << buildid << endl;
2463 : :
2464 : : // but is it unsatisfied the normal elfutils ways?
2465 : 16 : Dwarf* alt = dwarf_getalt (dbg);
2466 [ + - ]: 16 : if (alt == NULL)
2467 : : {
2468 : : // Yup, unsatisfied the normal way. Maybe we can satisfy it
2469 : : // from our own debuginfod database.
2470 : 16 : int alt_fd;
2471 : 16 : struct MHD_Response *r = 0;
2472 : 16 : try
2473 : : {
2474 [ + - ]: 16 : string artifacttype = "debuginfo";
2475 [ + - + - : 16 : r = handle_buildid (0, buildid, artifacttype, "", &alt_fd);
- + - + -
- ]
2476 : : }
2477 [ - - ]: 0 : catch (const reportable_exception& e)
2478 : : {
2479 : : // swallow exceptions
2480 : : }
2481 : :
2482 : : // NB: this is not actually recursive! This invokes the web-query
2483 : : // path, which cannot get back into the scan code paths.
2484 [ + - ]: 16 : if (r)
2485 : : {
2486 : : // Found it!
2487 : 16 : altdbg_fd = dup(alt_fd); // ok if this fails, downstream failures ok
2488 : 16 : alt = altdbg = dwarf_begin (altdbg_fd, DWARF_C_READ);
2489 : : // NB: must close this dwarf and this fd at the bottom of the function!
2490 : 16 : MHD_destroy_response (r); // will close alt_fd
2491 [ + - ]: 16 : if (alt)
2492 : 16 : dwarf_setalt (dbg, alt);
2493 : : }
2494 : : }
2495 : : else
2496 : : {
2497 : : // NB: dwarf_setalt(alt) inappropriate - already done!
2498 : : // NB: altdbg will stay 0 so nothing tries to redundantly dealloc.
2499 : : }
2500 : :
2501 [ + - ]: 16 : if (alt)
2502 : : {
2503 [ + - ]: 16 : if (verbose > 3)
2504 : 16 : obatched(clog) << "Resolved altdebug buildid=" << buildid << endl;
2505 : : }
2506 : : else // (alt == NULL) - signal possible presence of poor debuginfo
2507 : : {
2508 [ # # # # ]: 0 : debug_sourcefiles.insert("");
2509 [ # # ]: 0 : if (verbose > 3)
2510 : 0 : obatched(clog) << "Unresolved altdebug buildid=" << buildid << endl;
2511 : : }
2512 : : }
2513 : :
2514 : 82 : Dwarf_Off offset = 0;
2515 : 1076 : Dwarf_Off old_offset;
2516 : 1076 : size_t hsize;
2517 : :
2518 [ + + ]: 1076 : while (dwarf_nextcu (dbg, old_offset = offset, &offset, &hsize, NULL, NULL, NULL) == 0)
2519 : : {
2520 : 994 : Dwarf_Die cudie_mem;
2521 : 994 : Dwarf_Die *cudie = dwarf_offdie (dbg, old_offset + hsize, &cudie_mem);
2522 : :
2523 [ - + ]: 994 : if (cudie == NULL)
2524 : 8 : continue;
2525 [ + + ]: 994 : if (dwarf_tag (cudie) != DW_TAG_compile_unit)
2526 : 8 : continue;
2527 : :
2528 [ - + ]: 986 : const char *cuname = dwarf_diename(cudie) ?: "unknown";
2529 : :
2530 : 986 : Dwarf_Files *files;
2531 : 986 : size_t nfiles;
2532 [ - + ]: 986 : if (dwarf_getsrcfiles (cudie, &files, &nfiles) != 0)
2533 : 0 : continue;
2534 : :
2535 : : // extract DW_AT_comp_dir to resolve relative file names
2536 : 986 : const char *comp_dir = "";
2537 : 986 : const char *const *dirs;
2538 : 986 : size_t ndirs;
2539 [ + - ]: 986 : if (dwarf_getsrcdirs (files, &dirs, &ndirs) == 0 &&
2540 [ + - ]: 986 : dirs[0] != NULL)
2541 : 986 : comp_dir = dirs[0];
2542 [ - + ]: 986 : if (comp_dir == NULL)
2543 : 0 : comp_dir = "";
2544 : :
2545 [ + + ]: 986 : if (verbose > 3)
2546 : 122 : obatched(clog) << "searching for sources for cu=" << cuname << " comp_dir=" << comp_dir
2547 : 61 : << " #files=" << nfiles << " #dirs=" << ndirs << endl;
2548 : :
2549 [ + - - - ]: 986 : if (comp_dir[0] == '\0' && cuname[0] != '/')
2550 : : {
2551 : : // This is a common symptom for dwz-compressed debug files,
2552 : : // where the altdebug file cannot be resolved.
2553 [ # # ]: 0 : if (verbose > 3)
2554 : 0 : obatched(clog) << "skipping cu=" << cuname << " due to empty comp_dir" << endl;
2555 : 0 : continue;
2556 : : }
2557 : :
2558 [ + + ]: 15312 : for (size_t f = 1; f < nfiles; f++)
2559 : : {
2560 : 14326 : const char *hat = dwarf_filesrc (files, f, NULL, NULL);
2561 [ - + ]: 14326 : if (hat == NULL)
2562 : 0 : continue;
2563 : :
2564 [ + + - + ]: 27290 : if (string(hat) == "<built-in>") // gcc intrinsics, don't bother record
2565 : 0 : continue;
2566 : :
2567 [ + + ]: 28652 : string waldo;
2568 [ + + ]: 14326 : if (hat[0] == '/') // absolute
2569 [ - + ]: 8856 : waldo = (string (hat));
2570 [ + - ]: 5470 : else if (comp_dir[0] != '\0') // comp_dir relative
2571 [ - + + + : 9578 : waldo = (string (comp_dir) + string("/") + string (hat));
- + - + -
+ ]
2572 : : else
2573 : : {
2574 [ # # ]: 0 : if (verbose > 3)
2575 : 0 : obatched(clog) << "skipping hat=" << hat << " due to empty comp_dir" << endl;
2576 [ # # ]: 0 : continue;
2577 : : }
2578 : :
2579 : : // NB: this is the 'waldo' that a dbginfo client will have
2580 : : // to supply for us to give them the file The comp_dir
2581 : : // prefixing is a definite complication. Otherwise we'd
2582 : : // have to return a setof comp_dirs (one per CU!) with
2583 : : // corresponding filesrc[] names, instead of one absolute
2584 : : // resoved set. Maybe we'll have to do that anyway. XXX
2585 : :
2586 [ - + ]: 14326 : if (verbose > 4)
2587 [ # # ]: 0 : obatched(clog) << waldo
2588 [ # # ]: 0 : << (debug_sourcefiles.find(waldo)==debug_sourcefiles.end() ? " new" : " dup") << endl;
2589 : :
2590 [ + - ]: 14326 : debug_sourcefiles.insert (waldo);
2591 : : }
2592 : : }
2593 : :
2594 : 82 : dwarf_end(dbg);
2595 [ + + ]: 82 : if (altdbg)
2596 : 16 : dwarf_end(altdbg);
2597 [ + + ]: 82 : if (altdbg_fd >= 0)
2598 : 16 : close(altdbg_fd);
2599 : : }
2600 : :
2601 : :
2602 : :
2603 : : static void
2604 : 408 : elf_classify (int fd, bool &executable_p, bool &debuginfo_p, string &buildid, set<string>& debug_sourcefiles)
2605 : : {
2606 : 408 : Elf *elf = elf_begin (fd, ELF_C_READ_MMAP_PRIVATE, NULL);
2607 [ + - ]: 410 : if (elf == NULL)
2608 : : return;
2609 : :
2610 : 410 : try // catch our types of errors and clean up the Elf* object
2611 : : {
2612 [ + - + + ]: 410 : if (elf_kind (elf) != ELF_K_ELF)
2613 : : {
2614 [ + - ]: 251 : elf_end (elf);
2615 : 251 : return;
2616 : : }
2617 : :
2618 : 159 : GElf_Ehdr ehdr_storage;
2619 [ + - ]: 159 : GElf_Ehdr *ehdr = gelf_getehdr (elf, &ehdr_storage);
2620 [ - + ]: 158 : if (ehdr == NULL)
2621 : : {
2622 [ # # ]: 0 : elf_end (elf);
2623 : : return;
2624 : : }
2625 : 158 : auto elf_type = ehdr->e_type;
2626 : :
2627 : 158 : const void *build_id; // elfutils-owned memory
2628 [ + - ]: 158 : ssize_t sz = dwelf_elf_gnu_build_id (elf, & build_id);
2629 [ - + ]: 158 : if (sz <= 0)
2630 : : {
2631 : : // It's not a diagnostic-worthy error for an elf file to lack build-id.
2632 : : // It might just be very old.
2633 [ # # ]: 0 : elf_end (elf);
2634 : : return;
2635 : : }
2636 : :
2637 : : // build_id is a raw byte array; convert to hexadecimal *lowercase*
2638 : 158 : unsigned char* build_id_bytes = (unsigned char*) build_id;
2639 [ + + ]: 3317 : for (ssize_t idx=0; idx<sz; idx++)
2640 : : {
2641 [ + - ]: 3158 : buildid += "0123456789abcdef"[build_id_bytes[idx] >> 4];
2642 [ + - ]: 6311 : buildid += "0123456789abcdef"[build_id_bytes[idx] & 0xf];
2643 : : }
2644 : :
2645 : : // now decide whether it's an executable - namely, any allocatable section has
2646 : : // PROGBITS;
2647 [ + + ]: 159 : if (elf_type == ET_EXEC || elf_type == ET_DYN)
2648 : : {
2649 : 141 : size_t shnum;
2650 [ + - ]: 141 : int rc = elf_getshdrnum (elf, &shnum);
2651 [ - + ]: 139 : if (rc < 0)
2652 [ # # # # ]: 0 : throw elfutils_exception(rc, "getshdrnum");
2653 : :
2654 : 139 : executable_p = false;
2655 [ + + ]: 2638 : for (size_t sc = 0; sc < shnum; sc++)
2656 : : {
2657 [ + - ]: 2571 : Elf_Scn *scn = elf_getscn (elf, sc);
2658 [ - + ]: 2572 : if (scn == NULL)
2659 : 0 : continue;
2660 : :
2661 : 2572 : GElf_Shdr shdr_mem;
2662 [ + - ]: 2572 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_mem);
2663 [ - + ]: 2573 : if (shdr == NULL)
2664 : 0 : continue;
2665 : :
2666 : : // allocated (loadable / vm-addr-assigned) section with available content?
2667 [ + + + + ]: 2573 : if ((shdr->sh_type == SHT_PROGBITS) && (shdr->sh_flags & SHF_ALLOC))
2668 : : {
2669 [ - + ]: 74 : if (verbose > 4)
2670 [ # # # # : 0 : obatched(clog) << "executable due to SHF_ALLOC SHT_PROGBITS sc=" << sc << endl;
# # ]
2671 : 74 : executable_p = true;
2672 : 74 : break; // no need to keep looking for others
2673 : : }
2674 : : } // iterate over sections
2675 : : } // executable_p classification
2676 : :
2677 : : // now decide whether it's a debuginfo - namely, if it has any .debug* or .zdebug* sections
2678 : : // logic mostly stolen from fweimer@redhat.com's elfclassify drafts
2679 : 159 : size_t shstrndx;
2680 [ + - ]: 159 : int rc = elf_getshdrstrndx (elf, &shstrndx);
2681 [ - + ]: 159 : if (rc < 0)
2682 [ # # # # ]: 0 : throw elfutils_exception(rc, "getshdrstrndx");
2683 : :
2684 : : Elf_Scn *scn = NULL;
2685 : : bool symtab_p = false;
2686 : : bool bits_alloc_p = false;
2687 : 8191 : while (true)
2688 : : {
2689 [ + - ]: 4175 : scn = elf_nextscn (elf, scn);
2690 [ + + ]: 4173 : if (scn == NULL)
2691 : : break;
2692 : 4096 : GElf_Shdr shdr_storage;
2693 [ + - ]: 4096 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_storage);
2694 [ + - ]: 4078 : if (shdr == NULL)
2695 : : break;
2696 [ + - ]: 4078 : const char *section_name = elf_strptr (elf, shstrndx, shdr->sh_name);
2697 [ + - ]: 4098 : if (section_name == NULL)
2698 : : break;
2699 [ + + ]: 4098 : if (startswith (section_name, ".debug_line") ||
2700 [ - + ]: 4016 : startswith (section_name, ".zdebug_line"))
2701 : : {
2702 : 82 : debuginfo_p = true;
2703 : 82 : dwarf_extract_source_paths (elf, debug_sourcefiles);
2704 : 82 : break; // expecting only one .*debug_line, so no need to look for others
2705 : : }
2706 [ + + ]: 4016 : else if (startswith (section_name, ".debug_") ||
2707 [ + # ]: 3738 : startswith (section_name, ".zdebug_"))
2708 : : {
2709 : 258 : debuginfo_p = true;
2710 : : // NB: don't break; need to parse .debug_line for sources
2711 : : }
2712 [ + + ]: 3758 : else if (shdr->sh_type == SHT_SYMTAB)
2713 : : {
2714 : : symtab_p = true;
2715 : : }
2716 : 3747 : else if (shdr->sh_type != SHT_NOBITS
2717 [ + + ]: 3747 : && shdr->sh_type != SHT_NOTE
2718 [ + + ]: 1839 : && (shdr->sh_flags & SHF_ALLOC) != 0)
2719 : : {
2720 : 1568 : bits_alloc_p = true;
2721 : : }
2722 : 4016 : }
2723 : :
2724 : : // For more expansive elf/split-debuginfo classification, we
2725 : : // want to identify as debuginfo "strip -s"-produced files
2726 : : // without .debug_info* (like libicudata), but we don't want to
2727 : : // identify "strip -g" executables (with .symtab left there).
2728 [ - + ]: 159 : if (symtab_p && !bits_alloc_p)
2729 : 0 : debuginfo_p = true;
2730 : : }
2731 [ # # ]: 0 : catch (const reportable_exception& e)
2732 : : {
2733 [ # # ]: 0 : e.report(clog);
2734 : : }
2735 : 159 : elf_end (elf);
2736 : : }
2737 : :
2738 : :
2739 : : static void
2740 : 316 : scan_source_file (const string& rps, const stat_t& st,
2741 : : sqlite_ps& ps_upsert_buildids,
2742 : : sqlite_ps& ps_upsert_files,
2743 : : sqlite_ps& ps_upsert_de,
2744 : : sqlite_ps& ps_upsert_s,
2745 : : sqlite_ps& ps_query,
2746 : : sqlite_ps& ps_scan_done,
2747 : : unsigned& fts_cached,
2748 : : unsigned& fts_executable,
2749 : : unsigned& fts_debuginfo,
2750 : : unsigned& fts_sourcefiles)
2751 : : {
2752 : : /* See if we know of it already. */
2753 : 316 : int rc = ps_query
2754 : 316 : .reset()
2755 : 319 : .bind(1, rps)
2756 : 319 : .bind(2, st.st_mtime)
2757 : 319 : .step();
2758 : 319 : ps_query.reset();
2759 [ + + ]: 319 : if (rc == SQLITE_ROW) // i.e., a result, as opposed to DONE (no results)
2760 : : // no need to recheck a file/version we already know
2761 : : // specifically, no need to elf-begin a file we already determined is non-elf
2762 : : // (so is stored with buildid=NULL)
2763 : : {
2764 : 162 : fts_cached++;
2765 : 162 : return;
2766 : : }
2767 : :
2768 : 157 : bool executable_p = false, debuginfo_p = false; // E and/or D
2769 [ + - ]: 314 : string buildid;
2770 [ + - + + ]: 314 : set<string> sourcefiles;
2771 : :
2772 [ + - ]: 157 : int fd = open (rps.c_str(), O_RDONLY);
2773 : 157 : try
2774 : : {
2775 [ + - ]: 157 : if (fd >= 0)
2776 [ + - ]: 157 : elf_classify (fd, executable_p, debuginfo_p, buildid, sourcefiles);
2777 : : else
2778 [ # # # # : 0 : throw libc_exception(errno, string("open ") + rps);
# # # # #
# ]
2779 [ + - ]: 157 : add_metric ("scanned_bytes_total","source","file",
2780 [ + - + - : 471 : st.st_size);
+ - + - -
+ + - + -
- - - - -
- ]
2781 [ + - + - : 471 : inc_metric ("scanned_files_total","source","file");
+ - + - -
+ + - - -
- - ]
2782 : : }
2783 : : // NB: we catch exceptions here too, so that we can
2784 : : // cache the corrupt-elf case (!executable_p &&
2785 : : // !debuginfo_p) just below, just as if we had an
2786 : : // EPERM error from open(2).
2787 [ - - ]: 0 : catch (const reportable_exception& e)
2788 : : {
2789 [ - - ]: 0 : e.report(clog);
2790 : : }
2791 : :
2792 [ + - ]: 157 : if (fd >= 0)
2793 [ + - ]: 157 : close (fd);
2794 : :
2795 : : // register this file name in the interning table
2796 : 157 : ps_upsert_files
2797 [ + - ]: 157 : .reset()
2798 [ + - ]: 157 : .bind(1, rps)
2799 [ + - ]: 157 : .step_ok_done();
2800 : :
2801 [ + + ]: 157 : if (buildid == "")
2802 : : {
2803 : : // no point storing an elf file without buildid
2804 : 128 : executable_p = false;
2805 : 128 : debuginfo_p = false;
2806 : : }
2807 : : else
2808 : : {
2809 : : // register this build-id in the interning table
2810 : 29 : ps_upsert_buildids
2811 [ + - ]: 29 : .reset()
2812 [ + - ]: 29 : .bind(1, buildid)
2813 [ + - ]: 29 : .step_ok_done();
2814 : : }
2815 : :
2816 [ + + ]: 157 : if (executable_p)
2817 : 18 : fts_executable ++;
2818 [ + + ]: 157 : if (debuginfo_p)
2819 : 18 : fts_debuginfo ++;
2820 [ + + + + ]: 157 : if (executable_p || debuginfo_p)
2821 : : {
2822 : 29 : ps_upsert_de
2823 [ + - ]: 29 : .reset()
2824 [ + - ]: 29 : .bind(1, buildid)
2825 [ + + + - ]: 40 : .bind(2, debuginfo_p ? 1 : 0)
2826 [ + + + - ]: 40 : .bind(3, executable_p ? 1 : 0)
2827 [ + - ]: 29 : .bind(4, rps)
2828 [ + - ]: 29 : .bind(5, st.st_mtime)
2829 [ + - ]: 29 : .step_ok_done();
2830 : : }
2831 [ + + ]: 157 : if (executable_p)
2832 [ + - + - : 54 : inc_metric("found_executable_total","source","files");
+ - + - -
+ + - - -
- - ]
2833 [ + + ]: 157 : if (debuginfo_p)
2834 [ + - + - : 54 : inc_metric("found_debuginfo_total","source","files");
+ - + - -
+ + - - -
- - ]
2835 : :
2836 [ + + + - ]: 175 : if (sourcefiles.size() && buildid != "")
2837 : : {
2838 : 18 : fts_sourcefiles += sourcefiles.size();
2839 : :
2840 [ + + ]: 1479 : for (auto&& dwarfsrc : sourcefiles)
2841 : : {
2842 [ - + ]: 1461 : char *srp = realpath(dwarfsrc.c_str(), NULL);
2843 [ + + ]: 1461 : if (srp == NULL) // also if DWZ unresolved dwarfsrc=""
2844 : 18 : continue; // unresolvable files are not a serious problem
2845 : : // throw libc_exception(errno, "fts/file realpath " + srcpath);
2846 [ + - ]: 2886 : string srps = string(srp);
2847 : 1443 : free (srp);
2848 : :
2849 : 1443 : struct stat sfs;
2850 : 1443 : rc = stat(srps.c_str(), &sfs);
2851 [ - + ]: 1443 : if (rc != 0)
2852 [ - - ]: 18 : continue;
2853 : :
2854 [ + - ]: 1443 : if (verbose > 2)
2855 [ + - + - : 4329 : obatched(clog) << "recorded buildid=" << buildid << " file=" << srps
- - ]
2856 [ + - + - : 1443 : << " mtime=" << sfs.st_mtime
+ - + - ]
2857 [ + - + - : 1443 : << " as source " << dwarfsrc << endl;
+ - ]
2858 : :
2859 : 1443 : ps_upsert_files
2860 [ + - ]: 1443 : .reset()
2861 [ + - ]: 1443 : .bind(1, srps)
2862 [ + - ]: 1443 : .step_ok_done();
2863 : :
2864 : : // PR25548: store canonicalized dwarfsrc path
2865 [ + - + - ]: 2886 : string dwarfsrc_canon = canon_pathname (dwarfsrc);
2866 [ + + ]: 1443 : if (dwarfsrc_canon != dwarfsrc)
2867 : : {
2868 [ + + ]: 272 : if (verbose > 3)
2869 [ + - + - : 10 : obatched(clog) << "canonicalized src=" << dwarfsrc << " alias=" << dwarfsrc_canon << endl;
+ - + - +
- ]
2870 : : }
2871 : :
2872 : 1443 : ps_upsert_files
2873 [ + - ]: 1443 : .reset()
2874 [ + - ]: 1443 : .bind(1, dwarfsrc_canon)
2875 [ + - ]: 1443 : .step_ok_done();
2876 : :
2877 : 1443 : ps_upsert_s
2878 [ + - ]: 1443 : .reset()
2879 [ + - ]: 1443 : .bind(1, buildid)
2880 [ + - ]: 1443 : .bind(2, dwarfsrc_canon)
2881 [ + - ]: 1443 : .bind(3, srps)
2882 [ + - ]: 1443 : .bind(4, sfs.st_mtime)
2883 [ + - ]: 1443 : .step_ok_done();
2884 : :
2885 [ + - + - : 4329 : inc_metric("found_sourcerefs_total","source","files");
+ - + - -
+ + - + -
- - - - -
- ]
2886 : : }
2887 : : }
2888 : :
2889 : 157 : ps_scan_done
2890 [ + - ]: 157 : .reset()
2891 [ + - ]: 157 : .bind(1, rps)
2892 [ + - ]: 157 : .bind(2, st.st_mtime)
2893 [ + - ]: 157 : .bind(3, st.st_size)
2894 [ + - ]: 157 : .step_ok_done();
2895 : :
2896 [ + - ]: 157 : if (verbose > 2)
2897 [ + - + - ]: 471 : obatched(clog) << "recorded buildid=" << buildid << " file=" << rps
2898 [ + - + - : 157 : << " mtime=" << st.st_mtime << " atype="
+ - + - ]
2899 : : << (executable_p ? "E" : "")
2900 [ + - + + : 435 : << (debuginfo_p ? "D" : "") << endl;
+ - + + +
- + - ]
2901 : : }
2902 : :
2903 : :
2904 : :
2905 : :
2906 : :
2907 : : // Analyze given archive file of given age; record buildids / exec/debuginfo-ness of its
2908 : : // constituent files with given upsert statements.
2909 : : static void
2910 : 132 : archive_classify (const string& rps, string& archive_extension,
2911 : : sqlite_ps& ps_upsert_buildids, sqlite_ps& ps_upsert_files,
2912 : : sqlite_ps& ps_upsert_de, sqlite_ps& ps_upsert_sref, sqlite_ps& ps_upsert_sdef,
2913 : : time_t mtime,
2914 : : unsigned& fts_executable, unsigned& fts_debuginfo, unsigned& fts_sref, unsigned& fts_sdef,
2915 : : bool& fts_sref_complete_p)
2916 : : {
2917 : 132 : string archive_decoder = "/dev/null";
2918 [ + + ]: 334 : for (auto&& arch : scan_archives)
2919 [ + + ]: 202 : if (string_endswith(rps, arch.first))
2920 : : {
2921 [ + - ]: 132 : archive_extension = arch.first;
2922 [ + - ]: 334 : archive_decoder = arch.second;
2923 : : }
2924 : :
2925 : 132 : FILE* fp;
2926 : 132 : defer_dtor<FILE*,int>::dtor_fn dfn;
2927 [ + + ]: 132 : if (archive_decoder != "cat")
2928 : : {
2929 [ + - + - : 42 : string popen_cmd = archive_decoder + " " + shell_escape(rps);
+ - + + -
- - - ]
2930 [ + - ]: 14 : fp = popen (popen_cmd.c_str(), "r"); // "e" O_CLOEXEC?
2931 : 14 : dfn = pclose;
2932 [ - + ]: 14 : if (fp == NULL)
2933 [ # # # # : 0 : throw libc_exception (errno, string("popen ") + popen_cmd);
# # # # #
# ]
2934 : : }
2935 : : else
2936 : : {
2937 [ + - ]: 118 : fp = fopen (rps.c_str(), "r");
2938 : 118 : dfn = fclose;
2939 [ - + ]: 118 : if (fp == NULL)
2940 [ # # # # : 0 : throw libc_exception (errno, string("fopen ") + rps);
# # # # #
# ]
2941 : : }
2942 [ + + ]: 132 : defer_dtor<FILE*,int> fp_closer (fp, dfn);
2943 : :
2944 : 132 : struct archive *a;
2945 [ + - ]: 132 : a = archive_read_new();
2946 [ - + ]: 132 : if (a == NULL)
2947 [ # # # # ]: 0 : throw archive_exception("cannot create archive reader");
2948 : 132 : defer_dtor<struct archive*,int> archive_closer (a, archive_read_free);
2949 : :
2950 [ + - ]: 132 : int rc = archive_read_support_format_all(a);
2951 [ - + ]: 132 : if (rc != ARCHIVE_OK)
2952 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all formats");
2953 [ + - ]: 132 : rc = archive_read_support_filter_all(a);
2954 [ - + ]: 132 : if (rc != ARCHIVE_OK)
2955 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all filters");
2956 : :
2957 [ + - ]: 132 : rc = archive_read_open_FILE (a, fp);
2958 [ - + ]: 132 : if (rc != ARCHIVE_OK)
2959 [ # # # # ]: 0 : throw archive_exception(a, "cannot open archive from pipe");
2960 : :
2961 [ + + ]: 132 : if (verbose > 3)
2962 [ + - + - : 248 : obatched(clog) << "libarchive scanning " << rps << endl;
+ - ]
2963 : :
2964 : 951 : while(1) // parse archive entries
2965 : : {
2966 [ + - ]: 951 : if (interrupted)
2967 : : break;
2968 : :
2969 : 951 : try
2970 : : {
2971 : 951 : struct archive_entry *e;
2972 [ + - ]: 951 : rc = archive_read_next_header (a, &e);
2973 [ + + ]: 950 : if (rc != ARCHIVE_OK)
2974 : : break;
2975 : :
2976 [ + - + + ]: 818 : if (! S_ISREG(archive_entry_mode (e))) // skip non-files completely
2977 : 566 : continue;
2978 : :
2979 [ + - ]: 505 : string fn = canonicalized_archive_entry_pathname (e);
2980 : :
2981 [ + + ]: 253 : if (verbose > 3)
2982 [ + - + - : 466 : obatched(clog) << "libarchive checking " << fn << endl;
+ - - - ]
2983 : :
2984 : : // extract this file to a temporary file
2985 : 253 : char* tmppath = NULL;
2986 : 253 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
2987 [ - + ]: 253 : if (rc < 0)
2988 [ # # # # ]: 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
2989 : 0 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
2990 [ + - ]: 253 : int fd = mkstemp (tmppath);
2991 [ - + ]: 253 : if (fd < 0)
2992 [ # # # # ]: 0 : throw libc_exception (errno, "cannot create temporary file");
2993 : 253 : unlink (tmppath); // unlink now so OS will release the file as soon as we close the fd
2994 [ + + ]: 253 : defer_dtor<int,int> minifd_closer (fd, close);
2995 : :
2996 [ + - ]: 253 : rc = archive_read_data_into_fd (a, fd);
2997 [ - + ]: 253 : if (rc != ARCHIVE_OK)
2998 [ # # # # ]: 0 : throw archive_exception(a, "cannot extract file");
2999 : :
3000 : : // finally ... time to run elf_classify on this bad boy and update the database
3001 : 253 : bool executable_p = false, debuginfo_p = false;
3002 [ + - ]: 506 : string buildid;
3003 [ + - + + ]: 506 : set<string> sourcefiles;
3004 [ + - ]: 253 : elf_classify (fd, executable_p, debuginfo_p, buildid, sourcefiles);
3005 : : // NB: might throw
3006 : :
3007 [ + + ]: 253 : if (buildid != "") // intern buildid
3008 : : {
3009 : 130 : ps_upsert_buildids
3010 [ + - ]: 130 : .reset()
3011 [ + - ]: 130 : .bind(1, buildid)
3012 [ + - ]: 130 : .step_ok_done();
3013 : : }
3014 : :
3015 : 253 : ps_upsert_files // register this rpm constituent file name in interning table
3016 [ + - ]: 253 : .reset()
3017 [ + - ]: 253 : .bind(1, fn)
3018 [ + - ]: 253 : .step_ok_done();
3019 : :
3020 [ + + ]: 253 : if (sourcefiles.size() > 0) // sref records needed
3021 : : {
3022 : : // NB: we intern each source file once. Once raw, as it
3023 : : // appears in the DWARF file list coming back from
3024 : : // elf_classify() - because it'll end up in the
3025 : : // _norm.artifactsrc column. We don't also put another
3026 : : // version with a '.' at the front, even though that's
3027 : : // how rpm/cpio packs names, because we hide that from
3028 : : // the database for storage efficiency.
3029 : :
3030 [ + + ]: 234 : for (auto&& s : sourcefiles)
3031 : : {
3032 [ - + ]: 178 : if (s == "")
3033 : : {
3034 : 0 : fts_sref_complete_p = false;
3035 : 0 : continue;
3036 : : }
3037 : :
3038 : : // PR25548: store canonicalized source path
3039 : 178 : const string& dwarfsrc = s;
3040 [ + - ]: 356 : string dwarfsrc_canon = canon_pathname (dwarfsrc);
3041 [ + + ]: 178 : if (dwarfsrc_canon != dwarfsrc)
3042 : : {
3043 [ + - ]: 10 : if (verbose > 3)
3044 [ + - + - : 20 : obatched(clog) << "canonicalized src=" << dwarfsrc << " alias=" << dwarfsrc_canon << endl;
+ - + - +
- - - ]
3045 : : }
3046 : :
3047 : 178 : ps_upsert_files
3048 [ + - ]: 178 : .reset()
3049 [ + - ]: 178 : .bind(1, dwarfsrc_canon)
3050 [ + - ]: 178 : .step_ok_done();
3051 : :
3052 : 178 : ps_upsert_sref
3053 [ + - ]: 178 : .reset()
3054 [ + - ]: 178 : .bind(1, buildid)
3055 [ + - ]: 178 : .bind(2, dwarfsrc_canon)
3056 [ + - ]: 178 : .step_ok_done();
3057 : :
3058 [ + - ]: 178 : fts_sref ++;
3059 : : }
3060 : : }
3061 : :
3062 [ + + ]: 253 : if (executable_p)
3063 : 56 : fts_executable ++;
3064 [ + + ]: 253 : if (debuginfo_p)
3065 : 74 : fts_debuginfo ++;
3066 : :
3067 [ + + + + ]: 253 : if (executable_p || debuginfo_p)
3068 : : {
3069 : 130 : ps_upsert_de
3070 [ + - ]: 130 : .reset()
3071 [ + - ]: 130 : .bind(1, buildid)
3072 [ + + + - ]: 186 : .bind(2, debuginfo_p ? 1 : 0)
3073 [ + + + - ]: 204 : .bind(3, executable_p ? 1 : 0)
3074 [ + - ]: 130 : .bind(4, rps)
3075 [ + - ]: 130 : .bind(5, mtime)
3076 [ + - ]: 130 : .bind(6, fn)
3077 [ + - ]: 130 : .step_ok_done();
3078 : : }
3079 : : else // potential source - sdef record
3080 : : {
3081 : 123 : fts_sdef ++;
3082 : 123 : ps_upsert_sdef
3083 [ + - ]: 123 : .reset()
3084 [ + - ]: 123 : .bind(1, rps)
3085 [ + - ]: 123 : .bind(2, mtime)
3086 [ + - ]: 123 : .bind(3, fn)
3087 [ + - ]: 123 : .step_ok_done();
3088 : : }
3089 : :
3090 [ + - + + : 253 : if ((verbose > 2) && (executable_p || debuginfo_p))
+ + ]
3091 [ + - + - ]: 390 : obatched(clog) << "recorded buildid=" << buildid << " rpm=" << rps << " file=" << fn
3092 [ + - + - : 130 : << " mtime=" << mtime << " atype="
+ - + - +
- + - ]
3093 : : << (executable_p ? "E" : "")
3094 : : << (debuginfo_p ? "D" : "")
3095 [ + - + + : 260 : << " sourcefiles=" << sourcefiles.size() << endl;
+ - + + +
- + - + -
+ - ]
3096 : :
3097 : : }
3098 [ - - ]: 0 : catch (const reportable_exception& e)
3099 : : {
3100 [ - - ]: 0 : e.report(clog);
3101 : : }
3102 : : }
3103 : 132 : }
3104 : :
3105 : :
3106 : :
3107 : : // scan for archive files such as .rpm
3108 : : static void
3109 : 257 : scan_archive_file (const string& rps, const stat_t& st,
3110 : : sqlite_ps& ps_upsert_buildids,
3111 : : sqlite_ps& ps_upsert_files,
3112 : : sqlite_ps& ps_upsert_de,
3113 : : sqlite_ps& ps_upsert_sref,
3114 : : sqlite_ps& ps_upsert_sdef,
3115 : : sqlite_ps& ps_query,
3116 : : sqlite_ps& ps_scan_done,
3117 : : unsigned& fts_cached,
3118 : : unsigned& fts_executable,
3119 : : unsigned& fts_debuginfo,
3120 : : unsigned& fts_sref,
3121 : : unsigned& fts_sdef)
3122 : : {
3123 : : /* See if we know of it already. */
3124 : 257 : int rc = ps_query
3125 : 257 : .reset()
3126 : 262 : .bind(1, rps)
3127 : 262 : .bind(2, st.st_mtime)
3128 : 262 : .step();
3129 : 262 : ps_query.reset();
3130 [ + + ]: 262 : if (rc == SQLITE_ROW) // i.e., a result, as opposed to DONE (no results)
3131 : : // no need to recheck a file/version we already know
3132 : : // specifically, no need to parse this archive again, since we already have
3133 : : // it as a D or E or S record,
3134 : : // (so is stored with buildid=NULL)
3135 : : {
3136 : 130 : fts_cached ++;
3137 : 130 : return;
3138 : : }
3139 : :
3140 : : // intern the archive file name
3141 : 132 : ps_upsert_files
3142 : 132 : .reset()
3143 : 132 : .bind(1, rps)
3144 : 132 : .step_ok_done();
3145 : :
3146 : : // extract the archive contents
3147 : 132 : unsigned my_fts_executable = 0, my_fts_debuginfo = 0, my_fts_sref = 0, my_fts_sdef = 0;
3148 : 132 : bool my_fts_sref_complete_p = true;
3149 : 132 : try
3150 : : {
3151 [ + - ]: 264 : string archive_extension;
3152 : 132 : archive_classify (rps, archive_extension,
3153 : : ps_upsert_buildids, ps_upsert_files,
3154 : : ps_upsert_de, ps_upsert_sref, ps_upsert_sdef, // dalt
3155 [ + - ]: 132 : st.st_mtime,
3156 : : my_fts_executable, my_fts_debuginfo, my_fts_sref, my_fts_sdef,
3157 : : my_fts_sref_complete_p);
3158 [ + - + - : 396 : add_metric ("scanned_bytes_total","source",archive_extension + " archive",
- + - - ]
3159 [ + - + - : 528 : st.st_size);
+ - + - +
- - - -
- ]
3160 [ + - + - : 396 : inc_metric ("scanned_files_total","source",archive_extension + " archive");
+ - + - -
+ + - + -
- - - - ]
3161 [ + - + - : 396 : add_metric("found_debuginfo_total","source",archive_extension + " archive",
+ - - + -
- ]
3162 [ + - + - : 396 : my_fts_debuginfo);
+ - + - -
- ]
3163 [ + - + - : 396 : add_metric("found_executable_total","source",archive_extension + " archive",
+ - - + -
- ]
3164 [ + - + - : 396 : my_fts_executable);
+ - + - -
- ]
3165 [ + - + - : 396 : add_metric("found_sourcerefs_total","source",archive_extension + " archive",
+ - - + -
- ]
3166 [ + - + - : 396 : my_fts_sref);
+ - - + -
- - - ]
3167 : : }
3168 [ - - ]: 0 : catch (const reportable_exception& e)
3169 : : {
3170 [ - - ]: 0 : e.report(clog);
3171 : : }
3172 : :
3173 [ + - ]: 132 : if (verbose > 2)
3174 [ + - ]: 396 : obatched(clog) << "scanned archive=" << rps
3175 [ + - + - ]: 132 : << " mtime=" << st.st_mtime
3176 [ + - ]: 132 : << " executables=" << my_fts_executable
3177 [ + - + - ]: 132 : << " debuginfos=" << my_fts_debuginfo
3178 [ + - + - ]: 132 : << " srefs=" << my_fts_sref
3179 [ + - + - : 264 : << " sdefs=" << my_fts_sdef
+ - ]
3180 [ + - ]: 264 : << endl;
3181 : :
3182 : 132 : fts_executable += my_fts_executable;
3183 : 132 : fts_debuginfo += my_fts_debuginfo;
3184 : 132 : fts_sref += my_fts_sref;
3185 : 132 : fts_sdef += my_fts_sdef;
3186 : :
3187 [ + - ]: 132 : if (my_fts_sref_complete_p) // leave incomplete?
3188 : 132 : ps_scan_done
3189 : 132 : .reset()
3190 : 132 : .bind(1, rps)
3191 : 132 : .bind(2, st.st_mtime)
3192 : 132 : .bind(3, st.st_size)
3193 : 132 : .step_ok_done();
3194 : : }
3195 : :
3196 : :
3197 : :
3198 : : ////////////////////////////////////////////////////////////////////////
3199 : :
3200 : :
3201 : :
3202 : : // The thread that consumes file names off of the scanq. We hold
3203 : : // the persistent sqlite_ps's at this level and delegate file/archive
3204 : : // scanning to other functions.
3205 : : static void*
3206 : 112 : thread_main_scanner (void* arg)
3207 : : {
3208 : 112 : (void) arg;
3209 : :
3210 : : // all the prepared statements fit to use, the _f_ set:
3211 [ + - + - : 448 : sqlite_ps ps_f_upsert_buildids (db, "file-buildids-intern", "insert or ignore into " BUILDIDS "_buildids VALUES (NULL, ?);");
+ - + - -
- ]
3212 [ + - + - : 448 : sqlite_ps ps_f_upsert_files (db, "file-files-intern", "insert or ignore into " BUILDIDS "_files VALUES (NULL, ?);");
+ - + - +
- - - ]
3213 : 112 : sqlite_ps ps_f_upsert_de (db, "file-de-upsert",
3214 : : "insert or ignore into " BUILDIDS "_f_de "
3215 : : "(buildid, debuginfo_p, executable_p, file, mtime) "
3216 : : "values ((select id from " BUILDIDS "_buildids where hex = ?),"
3217 : : " ?,?,"
3218 [ + - + - : 336 : " (select id from " BUILDIDS "_files where name = ?), ?);");
+ - - + +
- - - ]
3219 : 112 : sqlite_ps ps_f_upsert_s (db, "file-s-upsert",
3220 : : "insert or ignore into " BUILDIDS "_f_s "
3221 : : "(buildid, artifactsrc, file, mtime) "
3222 : : "values ((select id from " BUILDIDS "_buildids where hex = ?),"
3223 : : " (select id from " BUILDIDS "_files where name = ?),"
3224 : : " (select id from " BUILDIDS "_files where name = ?),"
3225 [ + - + - : 336 : " ?);");
+ - - + +
- - - ]
3226 : 112 : sqlite_ps ps_f_query (db, "file-negativehit-find",
3227 : : "select 1 from " BUILDIDS "_file_mtime_scanned where sourcetype = 'F' "
3228 [ + - + - : 448 : "and file = (select id from " BUILDIDS "_files where name = ?) and mtime = ?;");
+ - + - +
- - - ]
3229 : 112 : sqlite_ps ps_f_scan_done (db, "file-scanned",
3230 : : "insert or ignore into " BUILDIDS "_file_mtime_scanned (sourcetype, file, mtime, size)"
3231 [ + - + - : 336 : "values ('F', (select id from " BUILDIDS "_files where name = ?), ?, ?);");
+ - - + +
- - - ]
3232 : :
3233 : : // and now for the _r_ set
3234 [ + - + - : 448 : sqlite_ps ps_r_upsert_buildids (db, "rpm-buildid-intern", "insert or ignore into " BUILDIDS "_buildids VALUES (NULL, ?);");
+ - + - +
- - - ]
3235 [ + - + - : 336 : sqlite_ps ps_r_upsert_files (db, "rpm-file-intern", "insert or ignore into " BUILDIDS "_files VALUES (NULL, ?);");
+ - - + +
- - - ]
3236 : 112 : sqlite_ps ps_r_upsert_de (db, "rpm-de-insert",
3237 : : "insert or ignore into " BUILDIDS "_r_de (buildid, debuginfo_p, executable_p, file, mtime, content) values ("
3238 : : "(select id from " BUILDIDS "_buildids where hex = ?), ?, ?, "
3239 : : "(select id from " BUILDIDS "_files where name = ?), ?, "
3240 [ + - + - : 336 : "(select id from " BUILDIDS "_files where name = ?));");
+ - - + +
- - - ]
3241 : 112 : sqlite_ps ps_r_upsert_sref (db, "rpm-sref-insert",
3242 : : "insert or ignore into " BUILDIDS "_r_sref (buildid, artifactsrc) values ("
3243 : : "(select id from " BUILDIDS "_buildids where hex = ?), "
3244 [ + - + - : 336 : "(select id from " BUILDIDS "_files where name = ?));");
+ - - + +
- - - ]
3245 : 112 : sqlite_ps ps_r_upsert_sdef (db, "rpm-sdef-insert",
3246 : : "insert or ignore into " BUILDIDS "_r_sdef (file, mtime, content) values ("
3247 : : "(select id from " BUILDIDS "_files where name = ?), ?,"
3248 [ + - + - : 336 : "(select id from " BUILDIDS "_files where name = ?));");
+ - - + +
- - - ]
3249 : 112 : sqlite_ps ps_r_query (db, "rpm-negativehit-query",
3250 : : "select 1 from " BUILDIDS "_file_mtime_scanned where "
3251 [ + - + - : 448 : "sourcetype = 'R' and file = (select id from " BUILDIDS "_files where name = ?) and mtime = ?;");
+ - + - +
- - - ]
3252 : 112 : sqlite_ps ps_r_scan_done (db, "rpm-scanned",
3253 : : "insert or ignore into " BUILDIDS "_file_mtime_scanned (sourcetype, file, mtime, size)"
3254 [ + - + - : 224 : "values ('R', (select id from " BUILDIDS "_files where name = ?), ?, ?);");
+ - - + +
- - - ]
3255 : :
3256 : :
3257 : 112 : unsigned fts_cached = 0, fts_executable = 0, fts_debuginfo = 0, fts_sourcefiles = 0;
3258 : 112 : unsigned fts_sref = 0, fts_sdef = 0;
3259 : :
3260 [ + - + - : 224 : add_metric("thread_count", "role", "scan", 1);
+ - + - -
+ - + + -
- - - - ]
3261 [ + - + - : 224 : add_metric("thread_busy", "role", "scan", 1);
+ - + - -
+ - + - -
- - ]
3262 [ + + ]: 590 : while (! interrupted)
3263 : : {
3264 [ + - ]: 844 : scan_payload p;
3265 : :
3266 [ + - + - : 956 : add_metric("thread_busy", "role", "scan", -1);
+ - + - -
+ - + + -
- - - - -
- ]
3267 [ + - ]: 478 : bool gotone = scanq.wait_front(p);
3268 [ + - + - : 956 : add_metric("thread_busy", "role", "scan", 1);
+ - + - -
+ - + + +
- - - - ]
3269 : :
3270 [ + + - + ]: 478 : if (! gotone) continue; // go back to waiting
3271 : :
3272 : 366 : try
3273 : : {
3274 : 366 : bool scan_archive = false;
3275 [ + + ]: 882 : for (auto&& arch : scan_archives)
3276 [ + + ]: 516 : if (string_endswith(p.first, arch.first))
3277 : 262 : scan_archive = true;
3278 : :
3279 [ + + ]: 366 : if (scan_archive)
3280 [ + - ]: 259 : scan_archive_file (p.first, p.second,
3281 : : ps_r_upsert_buildids,
3282 : : ps_r_upsert_files,
3283 : : ps_r_upsert_de,
3284 : : ps_r_upsert_sref,
3285 : : ps_r_upsert_sdef,
3286 : : ps_r_query,
3287 : : ps_r_scan_done,
3288 : : fts_cached,
3289 : : fts_executable,
3290 : : fts_debuginfo,
3291 : : fts_sref,
3292 : : fts_sdef);
3293 : :
3294 [ + + ]: 369 : if (scan_files) // NB: maybe "else if" ?
3295 [ + - ]: 319 : scan_source_file (p.first, p.second,
3296 : : ps_f_upsert_buildids,
3297 : : ps_f_upsert_files,
3298 : : ps_f_upsert_de,
3299 : : ps_f_upsert_s,
3300 : : ps_f_query,
3301 : : ps_f_scan_done,
3302 : : fts_cached, fts_executable, fts_debuginfo, fts_sourcefiles);
3303 : : }
3304 [ - - ]: 0 : catch (const reportable_exception& e)
3305 : : {
3306 [ - - ]: 0 : e.report(cerr);
3307 : : }
3308 : :
3309 [ + - ]: 369 : scanq.done_front(); // let idlers run
3310 : :
3311 : 366 : if (fts_cached || fts_executable || fts_debuginfo || fts_sourcefiles || fts_sref || fts_sdef)
3312 : : {} // NB: not just if a successful scan - we might have encountered -ENOSPC & failed
3313 [ + - + - : 732 : (void) statfs_free_enough_p(db_path, "database"); // report sqlite filesystem size
+ - ]
3314 [ + - + - : 732 : (void) statfs_free_enough_p(tmpdir, "tmpdir"); // this too, in case of fdcache/tmpfile usage
+ - ]
3315 : :
3316 : : // finished a scanning step -- not a "loop", because we just
3317 : : // consume the traversal loop's work, whenever
3318 [ + - + - : 1098 : inc_metric("thread_work_total","role","scan");
+ - + - -
+ + - + -
- - - - -
- ]
3319 : : }
3320 : :
3321 : :
3322 [ + - + - : 224 : add_metric("thread_busy", "role", "scan", -1);
+ - + - -
+ - + - -
- - ]
3323 : 224 : return 0;
3324 : : }
3325 : :
3326 : :
3327 : :
3328 : : // The thread that traverses all the source_paths and enqueues all the
3329 : : // matching files into the file/archive scan queue.
3330 : : static void
3331 : 51 : scan_source_paths()
3332 : : {
3333 : : // NB: fedora 31 glibc/fts(3) crashes inside fts_read() on empty
3334 : : // path list.
3335 [ + + ]: 51 : if (source_paths.empty())
3336 : 1 : return;
3337 : :
3338 : : // Turn the source_paths into an fts(3)-compatible char**. Since
3339 : : // source_paths[] does not change after argv processing, the
3340 : : // c_str()'s are safe to keep around awile.
3341 : 100 : vector<const char *> sps;
3342 [ + + ]: 136 : for (auto&& sp: source_paths)
3343 [ + - ]: 86 : sps.push_back(sp.c_str());
3344 [ + - - - ]: 50 : sps.push_back(NULL);
3345 : :
3346 [ + + + - ]: 93 : FTS *fts = fts_open ((char * const *)sps.data(),
3347 : : (traverse_logical ? FTS_LOGICAL : FTS_PHYSICAL|FTS_XDEV)
3348 : : | FTS_NOCHDIR /* multithreaded */,
3349 : : NULL);
3350 [ - + ]: 50 : if (fts == NULL)
3351 [ # # # # ]: 0 : throw libc_exception(errno, "cannot fts_open");
3352 [ + - ]: 50 : defer_dtor<FTS*,int> fts_cleanup (fts, fts_close);
3353 : :
3354 : 50 : struct timespec ts_start, ts_end;
3355 : 50 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
3356 : 50 : unsigned fts_scanned = 0, fts_regex = 0;
3357 : :
3358 : 747 : FTSENT *f;
3359 [ + - + + ]: 747 : while ((f = fts_read (fts)) != NULL)
3360 : : {
3361 [ + - ]: 697 : if (interrupted) break;
3362 : :
3363 [ - + ]: 697 : if (sigusr2 != forced_groom_count) // stop early if groom triggered
3364 : : {
3365 [ # # ]: 0 : scanq.clear(); // clear previously issued work for scanner threads
3366 : : break;
3367 : : }
3368 : :
3369 : 697 : fts_scanned ++;
3370 : :
3371 [ + - ]: 697 : if (verbose > 2)
3372 [ + - + - : 1394 : obatched(clog) << "fts traversing " << f->fts_path << endl;
+ - ]
3373 : :
3374 [ + + + + : 697 : switch (f->fts_info)
+ ]
3375 : : {
3376 : 366 : case FTS_F:
3377 : 366 : {
3378 : : /* Found a file. Convert it to an absolute path, so
3379 : : the buildid database does not have relative path
3380 : : names that are unresolvable from a subsequent run
3381 : : in a different cwd. */
3382 [ - + ]: 366 : char *rp = realpath(f->fts_path, NULL);
3383 [ - + ]: 366 : if (rp == NULL)
3384 : 0 : continue; // ignore dangling symlink or such
3385 [ + - ]: 366 : string rps = string(rp);
3386 : 366 : free (rp);
3387 : :
3388 [ + - ]: 366 : bool ri = !regexec (&file_include_regex, rps.c_str(), 0, 0, 0);
3389 [ + - ]: 366 : bool rx = !regexec (&file_exclude_regex, rps.c_str(), 0, 0, 0);
3390 [ - + ]: 366 : if (!ri || rx)
3391 : : {
3392 [ # # ]: 0 : if (verbose > 3)
3393 [ # # ]: 0 : obatched(clog) << "fts skipped by regex "
3394 [ # # # # : 0 : << (!ri ? "I" : "") << (rx ? "X" : "") << endl;
# # # # #
# ]
3395 : 0 : fts_regex ++;
3396 [ # # ]: 0 : if (!ri)
3397 [ # # # # : 0 : inc_metric("traversed_total","type","file-skipped-I");
# # # # #
# # # # #
# # # # ]
3398 [ # # ]: 0 : if (rx)
3399 [ # # # # : 0 : inc_metric("traversed_total","type","file-skipped-X");
# # # # #
# # # # #
# # ]
3400 : : }
3401 : : else
3402 : : {
3403 [ + - + - ]: 366 : scanq.push_back (make_pair(rps, *f->fts_statp));
3404 [ + - + - : 732 : inc_metric("traversed_total","type","file");
+ - + - -
+ - + - -
- - - - ]
3405 : : }
3406 : : }
3407 : 366 : break;
3408 : :
3409 : 1 : case FTS_ERR:
3410 : 1 : case FTS_NS:
3411 : : // report on some types of errors because they may reflect fixable misconfiguration
3412 : 1 : {
3413 [ + - + - : 2 : auto x = libc_exception(f->fts_errno, string("fts traversal ") + string(f->fts_path));
+ - + - -
+ - + + -
- - - - ]
3414 [ + - ]: 1 : x.report(cerr);
3415 : : }
3416 [ + - + - : 2 : inc_metric("traversed_total","type","error");
+ - + - -
+ - + - -
- - ]
3417 : 1 : break;
3418 : :
3419 : 6 : case FTS_SL: // ignore, but count because debuginfod -L would traverse these
3420 [ + - + - : 12 : inc_metric("traversed_total","type","symlink");
+ - + - -
+ - + - -
- - ]
3421 : 6 : break;
3422 : :
3423 : 162 : case FTS_D: // ignore
3424 [ + - + - : 324 : inc_metric("traversed_total","type","directory");
+ - + - -
+ - + - -
- - ]
3425 : 162 : break;
3426 : :
3427 : 162 : default: // ignore
3428 [ + - + - : 324 : inc_metric("traversed_total","type","other");
+ - + - -
+ - + - -
- - ]
3429 : 162 : break;
3430 : : }
3431 : : }
3432 : 50 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
3433 : 50 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
3434 : :
3435 [ + - + - : 150 : obatched(clog) << "fts traversed source paths in " << deltas << "s, scanned=" << fts_scanned
+ - + - ]
3436 [ + - + - : 50 : << ", regex-skipped=" << fts_regex << endl;
+ - ]
3437 : : }
3438 : :
3439 : :
3440 : : static void*
3441 : 28 : thread_main_fts_source_paths (void* arg)
3442 : : {
3443 : 28 : (void) arg; // ignore; we operate on global data
3444 : :
3445 [ + - + - : 56 : set_metric("thread_tid", "role","traverse", tid());
+ - - + -
+ - - -
- ]
3446 [ + - + - : 56 : add_metric("thread_count", "role", "traverse", 1);
+ - - + -
+ - - -
- ]
3447 : :
3448 : 28 : time_t last_rescan = 0;
3449 : :
3450 [ + - ]: 151 : while (! interrupted)
3451 : : {
3452 : 151 : sleep (1);
3453 : 151 : scanq.wait_idle(); // don't start a new traversal while scanners haven't finished the job
3454 : 151 : scanq.done_idle(); // release the hounds
3455 [ + + ]: 151 : if (interrupted) break;
3456 : :
3457 : 123 : time_t now = time(NULL);
3458 : 123 : bool rescan_now = false;
3459 [ + + ]: 123 : if (last_rescan == 0) // at least one initial rescan is documented even for -t0
3460 : 26 : rescan_now = true;
3461 [ + + + + ]: 123 : if (rescan_s > 0 && (long)now > (long)(last_rescan + rescan_s))
3462 : 4 : rescan_now = true;
3463 [ + + ]: 123 : if (sigusr1 != forced_rescan_count)
3464 : : {
3465 : 27 : forced_rescan_count = sigusr1;
3466 : 27 : rescan_now = true;
3467 : : }
3468 [ + + ]: 123 : if (rescan_now)
3469 : : {
3470 [ + - + - : 102 : set_metric("thread_busy", "role","traverse", 1);
+ - - + -
+ + - - -
- - ]
3471 : 51 : try
3472 : : {
3473 [ + - ]: 51 : scan_source_paths();
3474 : : }
3475 [ - - ]: 0 : catch (const reportable_exception& e)
3476 : : {
3477 [ - - ]: 0 : e.report(cerr);
3478 : : }
3479 : 51 : last_rescan = time(NULL); // NB: now was before scanning
3480 : : // finished a traversal loop
3481 [ + - + - : 153 : inc_metric("thread_work_total", "role","traverse");
+ - - + +
- - - -
- ]
3482 [ + - + - : 102 : set_metric("thread_busy", "role","traverse", 0);
+ - - + -
+ - - -
- ]
3483 : : }
3484 : : }
3485 : :
3486 : 28 : return 0;
3487 : : }
3488 : :
3489 : :
3490 : :
3491 : : ////////////////////////////////////////////////////////////////////////
3492 : :
3493 : : static void
3494 : 30 : database_stats_report()
3495 : : {
3496 : 30 : sqlite_ps ps_query (db, "database-overview",
3497 [ + - + - : 120 : "select label,quantity from " BUILDIDS "_stats");
+ - + - -
- ]
3498 : :
3499 [ + - + - ]: 60 : obatched(clog) << "database record counts:" << endl;
3500 : 630 : while (1)
3501 : : {
3502 [ + - ]: 330 : if (interrupted) break;
3503 [ + - ]: 330 : if (sigusr1 != forced_rescan_count) // stop early if scan triggered
3504 : : break;
3505 : :
3506 [ + - ]: 330 : int rc = ps_query.step();
3507 [ + + ]: 330 : if (rc == SQLITE_DONE) break;
3508 [ - + ]: 300 : if (rc != SQLITE_ROW)
3509 [ # # # # ]: 0 : throw sqlite_exception(rc, "step");
3510 : :
3511 [ + - ]: 300 : obatched(clog)
3512 [ + - - + : 300 : << ((const char*) sqlite3_column_text(ps_query, 0) ?: (const char*) "NULL")
+ - ]
3513 : : << " "
3514 [ + - + - : 600 : << (sqlite3_column_text(ps_query, 1) ?: (const unsigned char*) "NULL")
- + + - ]
3515 : 300 : << endl;
3516 : :
3517 [ + - + - : 600 : set_metric("groom", "statistic",
+ - ]
3518 [ + - ]: 300 : ((const char*) sqlite3_column_text(ps_query, 0) ?: (const char*) "NULL"),
3519 [ + - + - : 900 : (sqlite3_column_double(ps_query, 1)));
- + + - -
+ - + - -
- - ]
3520 : 300 : }
3521 : 30 : }
3522 : :
3523 : :
3524 : : // Do a round of database grooming that might take many minutes to run.
3525 : 30 : void groom()
3526 : : {
3527 [ + - ]: 60 : obatched(clog) << "grooming database" << endl;
3528 : :
3529 : 30 : struct timespec ts_start, ts_end;
3530 : 30 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
3531 : :
3532 : : // scan for files that have disappeared
3533 : 30 : sqlite_ps files (db, "check old files",
3534 : : "select distinct s.mtime, s.file, f.name from "
3535 : : BUILDIDS "_file_mtime_scanned s, " BUILDIDS "_files f "
3536 [ + - + - : 60 : "where f.id = s.file");
- + + - -
- ]
3537 : : // NB: Because _ftime_mtime_scanned can contain both F and
3538 : : // R records for the same file, this query would return duplicates if the
3539 : : // DISTINCT qualifier were not there.
3540 [ + - ]: 30 : files.reset();
3541 : :
3542 : : // DECISION TIME - we enumerate stale fileids/mtimes
3543 [ + - ]: 60 : deque<pair<int64_t,int64_t> > stale_fileid_mtime;
3544 : :
3545 : 30 : time_t time_start = time(NULL);
3546 : 130 : while(1)
3547 : : {
3548 : : // PR28514: limit grooming iteration to O(rescan time), to avoid
3549 : : // slow filesystem tests over many files locking out rescans for
3550 : : // too long.
3551 [ + + - + ]: 80 : if (rescan_s > 0 && (long)time(NULL) > (long)(time_start + rescan_s))
3552 : : {
3553 [ # # # # : 0 : inc_metric("groomed_total", "decision", "aborted");
# # # # #
# # # # #
# # ]
3554 : 0 : break;
3555 : : }
3556 : :
3557 [ + - ]: 80 : if (interrupted) break;
3558 : :
3559 [ + - ]: 80 : int rc = files.step();
3560 [ + + ]: 80 : if (rc != SQLITE_ROW)
3561 : : break;
3562 : :
3563 [ + - ]: 50 : int64_t mtime = sqlite3_column_int64 (files, 0);
3564 [ + - ]: 50 : int64_t fileid = sqlite3_column_int64 (files, 1);
3565 [ + - - + ]: 50 : const char* filename = ((const char*) sqlite3_column_text (files, 2) ?: "");
3566 : 50 : struct stat s;
3567 [ + - ]: 50 : bool reg_include = !regexec (&file_include_regex, filename, 0, 0, 0);
3568 [ + - ]: 50 : bool reg_exclude = !regexec (&file_exclude_regex, filename, 0, 0, 0);
3569 : :
3570 : 50 : rc = stat(filename, &s);
3571 [ - + - - : 50 : if ( (regex_groom && reg_exclude && !reg_include) || rc < 0 || (mtime != (int64_t) s.st_mtime) )
+ + - + ]
3572 : : {
3573 [ + - ]: 4 : if (verbose > 2)
3574 [ + - + - : 8 : obatched(clog) << "groom: stale file=" << filename << " mtime=" << mtime << endl;
+ - + - +
- ]
3575 [ + - ]: 4 : stale_fileid_mtime.push_back(make_pair(fileid,mtime));
3576 [ + - + - : 8 : inc_metric("groomed_total", "decision", "stale");
+ - + - -
+ - + + -
- - - - ]
3577 [ + - + - : 12 : set_metric("thread_work_pending","role","groom", stale_fileid_mtime.size());
+ - + - -
+ + - - -
- - ]
3578 : : }
3579 : : else
3580 [ + - + - : 92 : inc_metric("groomed_total", "decision", "fresh");
+ - + - -
+ - + - -
- - ]
3581 : :
3582 [ + - ]: 50 : if (sigusr1 != forced_rescan_count) // stop early if scan triggered
3583 : : break;
3584 : 50 : }
3585 [ + - ]: 30 : files.reset();
3586 : :
3587 : : // ACTION TIME
3588 : :
3589 : : // Now that we know which file/mtime tuples are stale, actually do
3590 : : // the deletion from the database. Doing this during the SELECT
3591 : : // iteration above results in undefined behaviour in sqlite, as per
3592 : : // https://www.sqlite.org/isolation.html
3593 : :
3594 : : // We could shuffle stale_fileid_mtime[] here. It'd let aborted
3595 : : // sequences of nuke operations resume at random locations, instead
3596 : : // of just starting over. But it doesn't matter much either way,
3597 : : // as long as we make progress.
3598 : :
3599 [ + - + - : 90 : sqlite_ps files_del_f_de (db, "nuke f_de", "delete from " BUILDIDS "_f_de where file = ? and mtime = ?");
+ - - + +
- - - ]
3600 [ + - + - : 90 : sqlite_ps files_del_r_de (db, "nuke r_de", "delete from " BUILDIDS "_r_de where file = ? and mtime = ?");
+ - - + +
- - - ]
3601 : 30 : sqlite_ps files_del_scan (db, "nuke f_m_s", "delete from " BUILDIDS "_file_mtime_scanned "
3602 [ + - + - : 90 : "where file = ? and mtime = ?");
+ - - + -
- ]
3603 : :
3604 [ + + ]: 34 : while (! stale_fileid_mtime.empty())
3605 : : {
3606 : 4 : auto stale = stale_fileid_mtime.front();
3607 : 4 : stale_fileid_mtime.pop_front();
3608 [ + - + - : 12 : set_metric("thread_work_pending","role","groom", stale_fileid_mtime.size());
+ - + - -
+ + - - +
- - - - ]
3609 : :
3610 : : // PR28514: limit grooming iteration to O(rescan time), to avoid
3611 : : // slow nuke_* queries over many files locking out rescans for too
3612 : : // long. We iterate over the files in random() sequence to avoid
3613 : : // partial checks going over the same set.
3614 [ - + - - ]: 4 : if (rescan_s > 0 && (long)time(NULL) > (long)(time_start + rescan_s))
3615 : : {
3616 [ # # # # : 0 : inc_metric("groomed_total", "action", "aborted");
# # # # #
# # # # #
# # ]
3617 : 0 : break;
3618 : : }
3619 : :
3620 [ + - ]: 4 : if (interrupted) break;
3621 : :
3622 : 4 : int64_t fileid = stale.first;
3623 : 4 : int64_t mtime = stale.second;
3624 [ + - + - : 4 : files_del_f_de.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
3625 [ + - + - : 4 : files_del_r_de.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
3626 [ + - + - : 4 : files_del_scan.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
3627 [ + - + - : 8 : inc_metric("groomed_total", "action", "cleaned");
+ - + - -
+ - + + -
- - - - ]
3628 : :
3629 [ + - ]: 4 : if (sigusr1 != forced_rescan_count) // stop early if scan triggered
3630 : : break;
3631 : : }
3632 : 30 : stale_fileid_mtime.clear(); // no need for this any longer
3633 [ + - + - : 90 : set_metric("thread_work_pending","role","groom", stale_fileid_mtime.size());
+ - + - -
+ + - + -
- - - - ]
3634 : :
3635 : : // delete buildids with no references in _r_de or _f_de tables;
3636 : : // cascades to _r_sref & _f_s records
3637 : 30 : sqlite_ps buildids_del (db, "nuke orphan buildids",
3638 : : "delete from " BUILDIDS "_buildids "
3639 : : "where not exists (select 1 from " BUILDIDS "_f_de d where " BUILDIDS "_buildids.id = d.buildid) "
3640 [ + - + - : 120 : "and not exists (select 1 from " BUILDIDS "_r_de d where " BUILDIDS "_buildids.id = d.buildid)");
+ - + - +
- - - ]
3641 [ + - + - ]: 30 : buildids_del.reset().step_ok_done();
3642 : :
3643 [ - + ]: 30 : if (interrupted) return;
3644 : :
3645 : : // NB: "vacuum" is too heavy for even daily runs: it rewrites the entire db, so is done as maxigroom -G
3646 [ + - + - : 120 : sqlite_ps g1 (db, "incremental vacuum", "pragma incremental_vacuum");
+ - + - +
- - - ]
3647 [ + - + - ]: 30 : g1.reset().step_ok_done();
3648 [ + - + - : 90 : sqlite_ps g2 (db, "optimize", "pragma optimize");
+ - - + +
- - - ]
3649 [ + - + - ]: 30 : g2.reset().step_ok_done();
3650 [ + - + - : 60 : sqlite_ps g3 (db, "wal checkpoint", "pragma wal_checkpoint=truncate");
+ - - + +
- - - ]
3651 [ + - + - ]: 30 : g3.reset().step_ok_done();
3652 : :
3653 [ + - ]: 30 : database_stats_report();
3654 : :
3655 [ + - + - : 60 : (void) statfs_free_enough_p(db_path, "database"); // report sqlite filesystem size
+ - ]
3656 : :
3657 [ + - ]: 30 : sqlite3_db_release_memory(db); // shrink the process if possible
3658 [ + - ]: 30 : sqlite3_db_release_memory(dbq); // ... for both connections
3659 [ + - ]: 30 : debuginfod_pool_groom(); // and release any debuginfod_client objects we've been holding onto
3660 : :
3661 [ + - ]: 30 : fdcache.limit(0,0,0,0); // release the fdcache contents
3662 [ + - ]: 30 : fdcache.limit(fdcache_fds, fdcache_mbs, fdcache_prefetch_fds, fdcache_prefetch_mbs); // restore status quo parameters
3663 : :
3664 : 30 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
3665 : 30 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
3666 : :
3667 [ + - + - : 60 : obatched(clog) << "groomed database in " << deltas << "s" << endl;
+ - + - ]
3668 : : }
3669 : :
3670 : :
3671 : : static void*
3672 : 31 : thread_main_groom (void* /*arg*/)
3673 : : {
3674 [ + - + - : 62 : set_metric("thread_tid", "role", "groom", tid());
+ - - + -
+ - - -
- ]
3675 [ + - + - : 62 : add_metric("thread_count", "role", "groom", 1);
+ - - + -
+ - - -
- ]
3676 : :
3677 : 31 : time_t last_groom = 0;
3678 : :
3679 : 279 : while (1)
3680 : : {
3681 : 155 : sleep (1);
3682 : 155 : scanq.wait_idle(); // PR25394: block scanners during grooming!
3683 [ + + ]: 155 : if (interrupted) break;
3684 : :
3685 : 124 : time_t now = time(NULL);
3686 : 124 : bool groom_now = false;
3687 [ + + ]: 124 : if (last_groom == 0) // at least one initial groom is documented even for -g0
3688 : 27 : groom_now = true;
3689 [ + + + + ]: 124 : if (groom_s > 0 && (long)now > (long)(last_groom + groom_s))
3690 : 4 : groom_now = true;
3691 [ + + ]: 124 : if (sigusr2 != forced_groom_count)
3692 : : {
3693 : 3 : forced_groom_count = sigusr2;
3694 : 3 : groom_now = true;
3695 : : }
3696 [ + + ]: 124 : if (groom_now)
3697 : : {
3698 [ + - + - : 60 : set_metric("thread_busy", "role", "groom", 1);
+ - - + -
+ + - - -
- - ]
3699 : 30 : try
3700 : : {
3701 [ + - ]: 30 : groom ();
3702 : : }
3703 [ - - ]: 0 : catch (const sqlite_exception& e)
3704 : : {
3705 [ - - - - : 0 : obatched(cerr) << e.message << endl;
- - ]
3706 : : }
3707 : 30 : last_groom = time(NULL); // NB: now was before grooming
3708 : : // finished a grooming loop
3709 [ + - + - : 90 : inc_metric("thread_work_total", "role", "groom");
+ - - + +
- - - -
- ]
3710 [ + - + - : 60 : set_metric("thread_busy", "role", "groom", 0);
+ - - + -
+ - - -
- ]
3711 : : }
3712 : :
3713 : 124 : scanq.done_idle();
3714 : 124 : }
3715 : :
3716 : 31 : return 0;
3717 : : }
3718 : :
3719 : :
3720 : : ////////////////////////////////////////////////////////////////////////
3721 : :
3722 : :
3723 : : static void
3724 : 32 : signal_handler (int /* sig */)
3725 : : {
3726 : 32 : interrupted ++;
3727 : :
3728 [ + + ]: 32 : if (db)
3729 : 31 : sqlite3_interrupt (db);
3730 [ + - ]: 32 : if (dbq)
3731 : 32 : sqlite3_interrupt (dbq);
3732 : :
3733 : : // NB: don't do anything else in here
3734 : 32 : }
3735 : :
3736 : : static void
3737 : 27 : sigusr1_handler (int /* sig */)
3738 : : {
3739 : 27 : sigusr1 ++;
3740 : : // NB: don't do anything else in here
3741 : 27 : }
3742 : :
3743 : : static void
3744 : 3 : sigusr2_handler (int /* sig */)
3745 : : {
3746 : 3 : sigusr2 ++;
3747 : : // NB: don't do anything else in here
3748 : 3 : }
3749 : :
3750 : :
3751 : : static void // error logging callback from libmicrohttpd internals
3752 : 0 : error_cb (void *arg, const char *fmt, va_list ap)
3753 : : {
3754 : 0 : (void) arg;
3755 [ # # # # : 0 : inc_metric("error_count","libmicrohttpd",fmt);
# # # # #
# # # #
# ]
3756 : 0 : char errmsg[512];
3757 : 0 : (void) vsnprintf (errmsg, sizeof(errmsg), fmt, ap); // ok if slightly truncated
3758 [ # # ]: 0 : obatched(cerr) << "libmicrohttpd error: " << errmsg; // MHD_DLOG calls already include \n
3759 : 0 : }
3760 : :
3761 : :
3762 : : // A user-defined sqlite function, to score the sharedness of the
3763 : : // prefix of two strings. This is used to compare candidate debuginfo
3764 : : // / source-rpm names, so that the closest match
3765 : : // (directory-topology-wise closest) is found. This is important in
3766 : : // case the same sref (source file name) is in many -debuginfo or
3767 : : // -debugsource RPMs, such as when multiple versions/releases of the
3768 : : // same package are in the database.
3769 : :
3770 : 131 : static void sqlite3_sharedprefix_fn (sqlite3_context* c, int argc, sqlite3_value** argv)
3771 : : {
3772 [ - + ]: 131 : if (argc != 2)
3773 : 0 : sqlite3_result_error(c, "expect 2 string arguments", -1);
3774 [ + - + + ]: 262 : else if ((sqlite3_value_type(argv[0]) != SQLITE_TEXT) ||
3775 : 131 : (sqlite3_value_type(argv[1]) != SQLITE_TEXT))
3776 : 3 : sqlite3_result_null(c);
3777 : : else
3778 : : {
3779 : 128 : const unsigned char* a = sqlite3_value_text (argv[0]);
3780 : 128 : const unsigned char* b = sqlite3_value_text (argv[1]);
3781 : 128 : int i = 0;
3782 [ + + + - : 10288 : while (*a != '\0' && *b != '\0' && *a++ == *b++)
+ + + + ]
3783 : 10048 : i++;
3784 : 128 : sqlite3_result_int (c, i);
3785 : : }
3786 : 131 : }
3787 : :
3788 : :
3789 : : int
3790 : 32 : main (int argc, char *argv[])
3791 : : {
3792 : 32 : (void) setlocale (LC_ALL, "");
3793 : 32 : (void) bindtextdomain (PACKAGE_TARNAME, LOCALEDIR);
3794 : 32 : (void) textdomain (PACKAGE_TARNAME);
3795 : :
3796 : : /* Tell the library which version we are expecting. */
3797 : 32 : elf_version (EV_CURRENT);
3798 : :
3799 [ + - - + ]: 64 : tmpdir = string(getenv("TMPDIR") ?: "/tmp");
3800 : :
3801 : : /* Set computed default values. */
3802 [ - + + - : 64 : db_path = string(getenv("HOME") ?: "/") + string("/.debuginfod.sqlite"); /* XDG? */
+ - - + +
- - + -
- ]
3803 : 32 : int rc = regcomp (& file_include_regex, ".*", REG_EXTENDED|REG_NOSUB); // match everything
3804 [ - + ]: 32 : if (rc != 0)
3805 : 0 : error (EXIT_FAILURE, 0, "regcomp failure: %d", rc);
3806 : 32 : rc = regcomp (& file_exclude_regex, "^$", REG_EXTENDED|REG_NOSUB); // match nothing
3807 [ - + ]: 32 : if (rc != 0)
3808 : 0 : error (EXIT_FAILURE, 0, "regcomp failure: %d", rc);
3809 : :
3810 : : // default parameters for fdcache are computed from system stats
3811 : 32 : struct statfs sfs;
3812 : 32 : rc = statfs(tmpdir.c_str(), &sfs);
3813 [ - + ]: 32 : if (rc < 0)
3814 : 0 : fdcache_mbs = 1024; // 1 gigabyte
3815 : : else
3816 : 32 : fdcache_mbs = sfs.f_bavail * sfs.f_bsize / 1024 / 1024 / 4; // 25% of free space
3817 : 32 : fdcache_mintmp = 25; // emergency flush at 25% remaining (75% full)
3818 : 32 : fdcache_prefetch = 64; // guesstimate storage is this much less costly than re-decompression
3819 : 32 : fdcache_fds = (concurrency + fdcache_prefetch) * 2;
3820 : :
3821 : : /* Parse and process arguments. */
3822 : 32 : int remaining;
3823 : 32 : argp_program_version_hook = print_version; // this works
3824 : 32 : (void) argp_parse (&argp, argc, argv, ARGP_IN_ORDER, &remaining, NULL);
3825 [ - + ]: 32 : if (remaining != argc)
3826 : 0 : error (EXIT_FAILURE, 0,
3827 : 0 : "unexpected argument: %s", argv[remaining]);
3828 : :
3829 [ + + + + : 32 : if (scan_archives.size()==0 && !scan_files && source_paths.size()>0)
- + ]
3830 [ # # ]: 0 : obatched(clog) << "warning: without -F -R -U -Z, ignoring PATHs" << endl;
3831 : :
3832 : 32 : fdcache.limit(fdcache_fds, fdcache_mbs, fdcache_prefetch_fds, fdcache_prefetch_mbs);
3833 : :
3834 : 32 : (void) signal (SIGPIPE, SIG_IGN); // microhttpd can generate it incidentally, ignore
3835 : 32 : (void) signal (SIGINT, signal_handler); // ^C
3836 : 32 : (void) signal (SIGHUP, signal_handler); // EOF
3837 : 32 : (void) signal (SIGTERM, signal_handler); // systemd
3838 : 32 : (void) signal (SIGUSR1, sigusr1_handler); // end-user
3839 : 32 : (void) signal (SIGUSR2, sigusr2_handler); // end-user
3840 : :
3841 : : /* Get database ready. */
3842 [ + + ]: 32 : if (! passive_p)
3843 : : {
3844 : 31 : rc = sqlite3_open_v2 (db_path.c_str(), &db, (SQLITE_OPEN_READWRITE
3845 : : |SQLITE_OPEN_URI
3846 : : |SQLITE_OPEN_PRIVATECACHE
3847 : : |SQLITE_OPEN_CREATE
3848 : : |SQLITE_OPEN_FULLMUTEX), /* thread-safe */
3849 : : NULL);
3850 [ - + ]: 31 : if (rc == SQLITE_CORRUPT)
3851 : : {
3852 : 0 : (void) unlink (db_path.c_str());
3853 : 0 : error (EXIT_FAILURE, 0,
3854 : : "cannot open %s, deleted database: %s", db_path.c_str(), sqlite3_errmsg(db));
3855 : : }
3856 [ - + ]: 31 : else if (rc)
3857 : : {
3858 : 0 : error (EXIT_FAILURE, 0,
3859 : : "cannot open %s, consider deleting database: %s", db_path.c_str(), sqlite3_errmsg(db));
3860 : : }
3861 : : }
3862 : :
3863 : : // open the readonly query variant
3864 : : // NB: PRIVATECACHE allows web queries to operate in parallel with
3865 : : // much other grooming/scanning operation.
3866 : 32 : rc = sqlite3_open_v2 (db_path.c_str(), &dbq, (SQLITE_OPEN_READONLY
3867 : : |SQLITE_OPEN_URI
3868 : : |SQLITE_OPEN_PRIVATECACHE
3869 : : |SQLITE_OPEN_FULLMUTEX), /* thread-safe */
3870 : : NULL);
3871 [ - + ]: 32 : if (rc)
3872 : : {
3873 : 0 : error (EXIT_FAILURE, 0,
3874 : : "cannot open %s, consider deleting database: %s", db_path.c_str(), sqlite3_errmsg(dbq));
3875 : : }
3876 : :
3877 : :
3878 [ + - ]: 64 : obatched(clog) << "opened database " << db_path
3879 [ + + + - : 33 : << (db?" rw":"") << (dbq?" ro":"") << endl;
- + + - +
- ]
3880 [ + - + - ]: 64 : obatched(clog) << "sqlite version " << sqlite3_version << endl;
3881 [ + + + - : 95 : obatched(clog) << "service mode " << (passive_p ? "passive":"active") << endl;
+ - ]
3882 : :
3883 : : // add special string-prefix-similarity function used in rpm sref/sdef resolution
3884 : 32 : rc = sqlite3_create_function(dbq, "sharedprefix", 2, SQLITE_UTF8, NULL,
3885 : : & sqlite3_sharedprefix_fn, NULL, NULL);
3886 [ - + ]: 32 : if (rc != SQLITE_OK)
3887 : 0 : error (EXIT_FAILURE, 0,
3888 : : "cannot create sharedprefix function: %s", sqlite3_errmsg(dbq));
3889 : :
3890 [ + + ]: 32 : if (! passive_p)
3891 : : {
3892 [ + + ]: 31 : if (verbose > 3)
3893 [ + - + - ]: 36 : obatched(clog) << "ddl: " << DEBUGINFOD_SQLITE_DDL << endl;
3894 : 31 : rc = sqlite3_exec (db, DEBUGINFOD_SQLITE_DDL, NULL, NULL, NULL);
3895 [ - + ]: 31 : if (rc != SQLITE_OK)
3896 : : {
3897 : 0 : error (EXIT_FAILURE, 0,
3898 : : "cannot run database schema ddl: %s", sqlite3_errmsg(db));
3899 : : }
3900 : : }
3901 : :
3902 : : // Start httpd server threads. Use a single dual-homed pool.
3903 [ + + + + ]: 90 : MHD_Daemon *d46 = MHD_start_daemon ((connection_pool ? 0 : MHD_USE_THREAD_PER_CONNECTION)
3904 : : #if MHD_VERSION >= 0x00095300
3905 : : | MHD_USE_INTERNAL_POLLING_THREAD
3906 : : #else
3907 : : | MHD_USE_SELECT_INTERNALLY
3908 : : #endif
3909 : : #ifdef MHD_USE_EPOLL
3910 : : | MHD_USE_EPOLL
3911 : : #endif
3912 : : | MHD_USE_DUAL_STACK
3913 : : #if MHD_VERSION >= 0x00095200
3914 : : | MHD_USE_ITC
3915 : : #endif
3916 : : | MHD_USE_DEBUG, /* report errors to stderr */
3917 : : http_port,
3918 : : NULL, NULL, /* default accept policy */
3919 : : handler_cb, NULL, /* handler callback */
3920 : : MHD_OPTION_EXTERNAL_LOGGER, error_cb, NULL,
3921 : : (connection_pool ? MHD_OPTION_THREAD_POOL_SIZE : MHD_OPTION_END),
3922 : : (connection_pool ? (int)connection_pool : MHD_OPTION_END),
3923 : : MHD_OPTION_END);
3924 : :
3925 [ - + ]: 32 : if (d46 == NULL)
3926 : : {
3927 : 0 : sqlite3 *database = db;
3928 : 0 : sqlite3 *databaseq = dbq;
3929 : 0 : db = dbq = 0; // for signal_handler not to freak
3930 : 0 : sqlite3_close (databaseq);
3931 : 0 : sqlite3_close (database);
3932 : 0 : error (EXIT_FAILURE, 0, "cannot start http server at port %d", http_port);
3933 : : }
3934 : :
3935 : 32 : obatched(clog) << "started http server on IPv4 IPv6 "
3936 [ + - + - : 32 : << "port=" << http_port << endl;
+ - ]
3937 : :
3938 : : // add maxigroom sql if -G given
3939 [ - + ]: 32 : if (maxigroom)
3940 : : {
3941 [ # # ]: 0 : obatched(clog) << "maxigrooming database, please wait." << endl;
3942 [ # # ]: 0 : extra_ddl.push_back("create index if not exists " BUILDIDS "_r_sref_arc on " BUILDIDS "_r_sref(artifactsrc);");
3943 [ # # ]: 0 : extra_ddl.push_back("delete from " BUILDIDS "_r_sdef where not exists (select 1 from " BUILDIDS "_r_sref b where " BUILDIDS "_r_sdef.content = b.artifactsrc);");
3944 [ # # ]: 0 : extra_ddl.push_back("drop index if exists " BUILDIDS "_r_sref_arc;");
3945 : :
3946 : : // NB: we don't maxigroom the _files interning table. It'd require a temp index on all the
3947 : : // tables that have file foreign-keys, which is a lot.
3948 : :
3949 : : // NB: with =delete, may take up 3x disk space total during vacuum process
3950 : : // vs. =off (only 2x but may corrupt database if program dies mid-vacuum)
3951 : : // vs. =wal (>3x observed, but safe)
3952 [ # # ]: 0 : extra_ddl.push_back("pragma journal_mode=delete;");
3953 [ # # ]: 0 : extra_ddl.push_back("vacuum;");
3954 [ # # ]: 0 : extra_ddl.push_back("pragma journal_mode=wal;");
3955 : : }
3956 : :
3957 : : // run extra -D sql if given
3958 [ + + ]: 32 : if (! passive_p)
3959 [ - + ]: 31 : for (auto&& i: extra_ddl)
3960 : : {
3961 [ # # ]: 0 : if (verbose > 1)
3962 [ # # # # ]: 0 : obatched(clog) << "extra ddl:\n" << i << endl;
3963 : 0 : rc = sqlite3_exec (db, i.c_str(), NULL, NULL, NULL);
3964 [ # # # # ]: 0 : if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)
3965 : 0 : error (0, 0,
3966 : : "warning: cannot run database extra ddl %s: %s", i.c_str(), sqlite3_errmsg(db));
3967 : :
3968 [ # # ]: 0 : if (maxigroom)
3969 [ # # ]: 0 : obatched(clog) << "maxigroomed database" << endl;
3970 : : }
3971 : :
3972 [ + + ]: 32 : if (! passive_p)
3973 [ + - + - ]: 62 : obatched(clog) << "search concurrency " << concurrency << endl;
3974 : 32 : obatched(clog) << "webapi connection pool " << connection_pool
3975 [ + - + + : 61 : << (connection_pool ? "" : " (unlimited)") << endl;
+ - + - ]
3976 [ + + ]: 32 : if (! passive_p)
3977 [ + - + - ]: 62 : obatched(clog) << "rescan time " << rescan_s << endl;
3978 [ + - + - ]: 64 : obatched(clog) << "fdcache fds " << fdcache_fds << endl;
3979 [ + - + - ]: 64 : obatched(clog) << "fdcache mbs " << fdcache_mbs << endl;
3980 [ + - + - ]: 64 : obatched(clog) << "fdcache prefetch " << fdcache_prefetch << endl;
3981 [ + - + - ]: 64 : obatched(clog) << "fdcache tmpdir " << tmpdir << endl;
3982 [ + - + - ]: 64 : obatched(clog) << "fdcache tmpdir min% " << fdcache_mintmp << endl;
3983 [ + + ]: 32 : if (! passive_p)
3984 [ + - + - ]: 62 : obatched(clog) << "groom time " << groom_s << endl;
3985 [ + - + - ]: 64 : obatched(clog) << "prefetch fds " << fdcache_prefetch_fds << endl;
3986 [ + - + - ]: 64 : obatched(clog) << "prefetch mbs " << fdcache_prefetch_mbs << endl;
3987 [ + - + - ]: 64 : obatched(clog) << "forwarded ttl limit " << forwarded_ttl_limit << endl;
3988 : :
3989 [ + + ]: 32 : if (scan_archives.size()>0)
3990 : : {
3991 : 20 : obatched ob(clog);
3992 [ + - ]: 20 : auto& o = ob << "accepting archive types ";
3993 [ + + ]: 62 : for (auto&& arch : scan_archives)
3994 [ + - + - : 42 : o << arch.first << "(" << arch.second << ") ";
+ - + - ]
3995 [ + - ]: 20 : o << endl;
3996 : : }
3997 : 32 : const char* du = getenv(DEBUGINFOD_URLS_ENV_VAR);
3998 [ + + + + ]: 32 : if (du && du[0] != '\0') // set to non-empty string?
3999 [ + - + - ]: 14 : obatched(clog) << "upstream debuginfod servers: " << du << endl;
4000 : :
4001 [ + + ]: 64 : vector<pthread_t> all_threads;
4002 : :
4003 [ + + ]: 32 : if (! passive_p)
4004 : : {
4005 : 31 : pthread_t pt;
4006 : 31 : rc = pthread_create (& pt, NULL, thread_main_groom, NULL);
4007 [ - + ]: 31 : if (rc)
4008 : 0 : error (EXIT_FAILURE, rc, "cannot spawn thread to groom database\n");
4009 : : else
4010 : : {
4011 : : #ifdef HAVE_PTHREAD_SETNAME_NP
4012 : 31 : (void) pthread_setname_np (pt, "groom");
4013 : : #endif
4014 [ + - ]: 31 : all_threads.push_back(pt);
4015 : : }
4016 : :
4017 [ + + + + ]: 31 : if (scan_files || scan_archives.size() > 0)
4018 : : {
4019 : 28 : rc = pthread_create (& pt, NULL, thread_main_fts_source_paths, NULL);
4020 [ - + ]: 28 : if (rc)
4021 : 0 : error (EXIT_FAILURE, rc, "cannot spawn thread to traverse source paths\n");
4022 : : #ifdef HAVE_PTHREAD_SETNAME_NP
4023 : 28 : (void) pthread_setname_np (pt, "traverse");
4024 : : #endif
4025 [ + - ]: 28 : all_threads.push_back(pt);
4026 : :
4027 [ + + ]: 140 : for (unsigned i=0; i<concurrency; i++)
4028 : : {
4029 : 112 : rc = pthread_create (& pt, NULL, thread_main_scanner, NULL);
4030 [ - + ]: 112 : if (rc)
4031 : 0 : error (EXIT_FAILURE, rc, "cannot spawn thread to scan source files / archives\n");
4032 : : #ifdef HAVE_PTHREAD_SETNAME_NP
4033 : 112 : (void) pthread_setname_np (pt, "scan");
4034 : : #endif
4035 [ + - ]: 112 : all_threads.push_back(pt);
4036 : : }
4037 : : }
4038 : : }
4039 : :
4040 : : /* Trivial main loop! */
4041 [ + - + - : 64 : set_metric("ready", 1);
- - ]
4042 [ + + ]: 94 : while (! interrupted)
4043 [ + - ]: 62 : pause ();
4044 [ + - ]: 32 : scanq.nuke(); // wake up any remaining scanq-related threads, let them die
4045 [ + - + - : 64 : set_metric("ready", 0);
+ - ]
4046 : :
4047 [ + - ]: 32 : if (verbose)
4048 [ + - + - : 64 : obatched(clog) << "stopping" << endl;
- - ]
4049 : :
4050 : : /* Join all our threads. */
4051 [ + + ]: 203 : for (auto&& it : all_threads)
4052 [ + - ]: 171 : pthread_join (it, NULL);
4053 : :
4054 : : /* Stop all the web service threads. */
4055 [ + - ]: 32 : if (d46) MHD_stop_daemon (d46);
4056 : :
4057 [ + + ]: 32 : if (! passive_p)
4058 : : {
4059 : : /* With all threads known dead, we can clean up the global resources. */
4060 [ + - ]: 31 : rc = sqlite3_exec (db, DEBUGINFOD_SQLITE_CLEANUP_DDL, NULL, NULL, NULL);
4061 [ - + ]: 31 : if (rc != SQLITE_OK)
4062 : : {
4063 [ # # # # ]: 0 : error (0, 0,
4064 : : "warning: cannot run database cleanup ddl: %s", sqlite3_errmsg(db));
4065 : : }
4066 : : }
4067 : :
4068 [ + - ]: 32 : debuginfod_pool_groom ();
4069 : :
4070 : : // NB: no problem with unconditional free here - an earlier failed regcomp would exit program
4071 [ + - ]: 32 : (void) regfree (& file_include_regex);
4072 [ + - ]: 32 : (void) regfree (& file_exclude_regex);
4073 : :
4074 : 32 : sqlite3 *database = db;
4075 : 32 : sqlite3 *databaseq = dbq;
4076 : 32 : db = dbq = 0; // for signal_handler not to freak
4077 [ + - ]: 32 : (void) sqlite3_close (databaseq);
4078 [ + + ]: 32 : if (! passive_p)
4079 [ + - ]: 31 : (void) sqlite3_close (database);
4080 : :
4081 [ + + ]: 32 : return 0;
4082 : : }
|
