Line data Source code
1 : /* Debuginfo-over-http server.
2 : Copyright (C) 2019-2020 Red Hat, Inc.
3 : This file is part of elfutils.
4 :
5 : This file is free software; you can redistribute it and/or modify
6 : it under the terms of the GNU General Public License as published by
7 : the Free Software Foundation; either version 3 of the License, or
8 : (at your option) any later version.
9 :
10 : elfutils is distributed in the hope that it will be useful, but
11 : WITHOUT ANY WARRANTY; without even the implied warranty of
12 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : GNU General Public License for more details.
14 :
15 : You should have received a copy of the GNU General Public License
16 : along with this program. If not, see <http://www.gnu.org/licenses/>. */
17 :
18 :
19 : /* cargo-cult from libdwfl linux-kernel-modules.c */
20 : /* In case we have a bad fts we include this before config.h because it
21 : can't handle _FILE_OFFSET_BITS.
22 : Everything we need here is fine if its declarations just come first.
23 : Also, include sys/types.h before fts. On some systems fts.h is not self
24 : contained. */
25 : #ifdef BAD_FTS
26 : #include <sys/types.h>
27 : #include <fts.h>
28 : #endif
29 :
30 : #ifdef HAVE_CONFIG_H
31 : #include "config.h"
32 : #endif
33 :
34 : extern "C" {
35 : #include "printversion.h"
36 : }
37 :
38 : #include "debuginfod.h"
39 : #include <dwarf.h>
40 :
41 : #include <argp.h>
42 : #ifdef __GNUC__
43 : #undef __attribute__ /* glibc bug - rhbz 1763325 */
44 : #endif
45 :
46 : #include <unistd.h>
47 : #include <stdlib.h>
48 : #include <error.h>
49 : // #include <libintl.h> // not until it supports C++ << better
50 : #include <locale.h>
51 : #include <pthread.h>
52 : #include <signal.h>
53 : #include <sys/stat.h>
54 : #include <sys/time.h>
55 : #include <sys/vfs.h>
56 : #include <unistd.h>
57 : #include <fcntl.h>
58 : #include <netdb.h>
59 :
60 :
61 : /* If fts.h is included before config.h, its indirect inclusions may not
62 : give us the right LFS aliases of these functions, so map them manually. */
63 : #ifdef BAD_FTS
64 : #ifdef _FILE_OFFSET_BITS
65 : #define open open64
66 : #define fopen fopen64
67 : #endif
68 : #else
69 : #include <sys/types.h>
70 : #include <fts.h>
71 : #endif
72 :
73 : #include <cstring>
74 : #include <vector>
75 : #include <set>
76 : #include <map>
77 : #include <string>
78 : #include <iostream>
79 : #include <iomanip>
80 : #include <ostream>
81 : #include <sstream>
82 : #include <mutex>
83 : #include <deque>
84 : #include <condition_variable>
85 : #include <thread>
86 : // #include <regex> // on rhel7 gcc 4.8, not competent
87 : #include <regex.h>
88 : // #include <algorithm>
89 : using namespace std;
90 :
91 : #include <gelf.h>
92 : #include <libdwelf.h>
93 :
94 : #include <microhttpd.h>
95 : #include <curl/curl.h>
96 : #include <archive.h>
97 : #include <archive_entry.h>
98 : #include <sqlite3.h>
99 :
100 : #ifdef __linux__
101 : #include <sys/syscall.h>
102 : #endif
103 :
104 : #ifdef __linux__
105 : #define tid() syscall(SYS_gettid)
106 : #else
107 : #define tid() pthread_self()
108 : #endif
109 :
110 :
111 : inline bool
112 260 : string_endswith(const string& haystack, const string& needle)
113 : {
114 520 : return (haystack.size() >= needle.size() &&
115 260 : equal(haystack.end()-needle.size(), haystack.end(),
116 260 : needle.begin()));
117 : }
118 :
119 :
120 : // Roll this identifier for every sqlite schema incompatiblity.
121 : #define BUILDIDS "buildids9"
122 :
123 : #if SQLITE_VERSION_NUMBER >= 3008000
124 : #define WITHOUT_ROWID "without rowid"
125 : #else
126 : #define WITHOUT_ROWID ""
127 : #endif
128 :
129 : static const char DEBUGINFOD_SQLITE_DDL[] =
130 : "pragma foreign_keys = on;\n"
131 : "pragma synchronous = 0;\n" // disable fsync()s - this cache is disposable across a machine crash
132 : "pragma journal_mode = wal;\n" // https://sqlite.org/wal.html
133 : "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file
134 : "pragma journal_size_limit = 0;\n" // limit steady state file (between grooming, which also =truncate's)
135 : "pragma auto_vacuum = incremental;\n" // https://sqlite.org/pragma.html
136 : "pragma busy_timeout = 1000;\n" // https://sqlite.org/pragma.html
137 : // NB: all these are overridable with -D option
138 :
139 : // Normalization table for interning file names
140 : "create table if not exists " BUILDIDS "_files (\n"
141 : " id integer primary key not null,\n"
142 : " name text unique not null\n"
143 : " );\n"
144 : // Normalization table for interning buildids
145 : "create table if not exists " BUILDIDS "_buildids (\n"
146 : " id integer primary key not null,\n"
147 : " hex text unique not null);\n"
148 : // Track the completion of scanning of a given file & sourcetype at given time
149 : "create table if not exists " BUILDIDS "_file_mtime_scanned (\n"
150 : " mtime integer not null,\n"
151 : " file integer not null,\n"
152 : " size integer not null,\n" // in bytes
153 : " sourcetype text(1) not null\n"
154 : " check (sourcetype IN ('F', 'R')),\n"
155 : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
156 : " primary key (file, mtime, sourcetype)\n"
157 : " ) " WITHOUT_ROWID ";\n"
158 : "create table if not exists " BUILDIDS "_f_de (\n"
159 : " buildid integer not null,\n"
160 : " debuginfo_p integer not null,\n"
161 : " executable_p integer not null,\n"
162 : " file integer not null,\n"
163 : " mtime integer not null,\n"
164 : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
165 : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
166 : " primary key (buildid, file, mtime)\n"
167 : " ) " WITHOUT_ROWID ";\n"
168 : "create table if not exists " BUILDIDS "_f_s (\n"
169 : " buildid integer not null,\n"
170 : " artifactsrc integer not null,\n"
171 : " file integer not null,\n" // NB: not necessarily entered into _mtime_scanned
172 : " mtime integer not null,\n"
173 : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
174 : " foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
175 : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
176 : " primary key (buildid, artifactsrc, file, mtime)\n"
177 : " ) " WITHOUT_ROWID ";\n"
178 : "create table if not exists " BUILDIDS "_r_de (\n"
179 : " buildid integer not null,\n"
180 : " debuginfo_p integer not null,\n"
181 : " executable_p integer not null,\n"
182 : " file integer not null,\n"
183 : " mtime integer not null,\n"
184 : " content integer not null,\n"
185 : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
186 : " foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
187 : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
188 : " primary key (buildid, debuginfo_p, executable_p, file, content, mtime)\n"
189 : " ) " WITHOUT_ROWID ";\n"
190 : "create table if not exists " BUILDIDS "_r_sref (\n" // outgoing dwarf sourcefile references from rpm
191 : " buildid integer not null,\n"
192 : " artifactsrc integer not null,\n"
193 : " foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
194 : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
195 : " primary key (buildid, artifactsrc)\n"
196 : " ) " WITHOUT_ROWID ";\n"
197 : "create table if not exists " BUILDIDS "_r_sdef (\n" // rpm contents that may satisfy sref
198 : " file integer not null,\n"
199 : " mtime integer not null,\n"
200 : " content integer not null,\n"
201 : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
202 : " foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
203 : " primary key (content, file, mtime)\n"
204 : " ) " WITHOUT_ROWID ";\n"
205 : // create views to glue together some of the above tables, for webapi D queries
206 : "create view if not exists " BUILDIDS "_query_d as \n"
207 : "select\n"
208 : " b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"
209 : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_f_de n\n"
210 : " where b.id = n.buildid and f0.id = n.file and n.debuginfo_p = 1\n"
211 : "union all select\n"
212 : " b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"
213 : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_r_de n\n"
214 : " where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.debuginfo_p = 1\n"
215 : ";"
216 : // ... and for E queries
217 : "create view if not exists " BUILDIDS "_query_e as \n"
218 : "select\n"
219 : " b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"
220 : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_f_de n\n"
221 : " where b.id = n.buildid and f0.id = n.file and n.executable_p = 1\n"
222 : "union all select\n"
223 : " b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"
224 : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_r_de n\n"
225 : " where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.executable_p = 1\n"
226 : ";"
227 : // ... and for S queries
228 : "create view if not exists " BUILDIDS "_query_s as \n"
229 : "select\n"
230 : " b.hex as buildid, fs.name as artifactsrc, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1, null as source0ref\n"
231 : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files fs, " BUILDIDS "_f_s n\n"
232 : " where b.id = n.buildid and f0.id = n.file and fs.id = n.artifactsrc\n"
233 : "union all select\n"
234 : " b.hex as buildid, f1.name as artifactsrc, 'R' as sourcetype, f0.name as source0, sd.mtime as mtime, f1.name as source1, fsref.name as source0ref\n"
235 : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_files fsref, "
236 : " " BUILDIDS "_r_sdef sd, " BUILDIDS "_r_sref sr, " BUILDIDS "_r_de sde\n"
237 : " where b.id = sr.buildid and f0.id = sd.file and fsref.id = sde.file and f1.id = sd.content\n"
238 : " and sr.artifactsrc = sd.content and sde.buildid = sr.buildid\n"
239 : ";"
240 : // and for startup overview counts
241 : "drop view if exists " BUILDIDS "_stats;\n"
242 : "create view if not exists " BUILDIDS "_stats as\n"
243 : " select 'file d/e' as label,count(*) as quantity from " BUILDIDS "_f_de\n"
244 : "union all select 'file s',count(*) from " BUILDIDS "_f_s\n"
245 : "union all select 'archive d/e',count(*) from " BUILDIDS "_r_de\n"
246 : "union all select 'archive sref',count(*) from " BUILDIDS "_r_sref\n"
247 : "union all select 'archive sdef',count(*) from " BUILDIDS "_r_sdef\n"
248 : "union all select 'buildids',count(*) from " BUILDIDS "_buildids\n"
249 : "union all select 'filenames',count(*) from " BUILDIDS "_files\n"
250 : "union all select 'files scanned (#)',count(*) from " BUILDIDS "_file_mtime_scanned\n"
251 : "union all select 'files scanned (mb)',coalesce(sum(size)/1024/1024,0) from " BUILDIDS "_file_mtime_scanned\n"
252 : #if SQLITE_VERSION_NUMBER >= 3016000
253 : "union all select 'index db size (mb)',page_count*page_size/1024/1024 as size FROM pragma_page_count(), pragma_page_size()\n"
254 : #endif
255 : ";\n"
256 :
257 : // schema change history & garbage collection
258 : //
259 : // XXX: we could have migration queries here to bring prior-schema
260 : // data over instead of just dropping it.
261 : //
262 : // buildids9: widen the mtime_scanned table
263 : "" // <<< we are here
264 : // buildids8: slim the sref table
265 : "drop table if exists buildids8_f_de;\n"
266 : "drop table if exists buildids8_f_s;\n"
267 : "drop table if exists buildids8_r_de;\n"
268 : "drop table if exists buildids8_r_sref;\n"
269 : "drop table if exists buildids8_r_sdef;\n"
270 : "drop table if exists buildids8_file_mtime_scanned;\n"
271 : "drop table if exists buildids8_files;\n"
272 : "drop table if exists buildids8_buildids;\n"
273 : // buildids7: separate _norm table into dense subtype tables
274 : "drop table if exists buildids7_f_de;\n"
275 : "drop table if exists buildids7_f_s;\n"
276 : "drop table if exists buildids7_r_de;\n"
277 : "drop table if exists buildids7_r_sref;\n"
278 : "drop table if exists buildids7_r_sdef;\n"
279 : "drop table if exists buildids7_file_mtime_scanned;\n"
280 : "drop table if exists buildids7_files;\n"
281 : "drop table if exists buildids7_buildids;\n"
282 : // buildids6: drop bolo/rfolo again, represent sources / rpmcontents in main table
283 : "drop table if exists buildids6_norm;\n"
284 : "drop table if exists buildids6_files;\n"
285 : "drop table if exists buildids6_buildids;\n"
286 : "drop view if exists buildids6;\n"
287 : // buildids5: redefine srcfile1 column to be '.'-less (for rpms)
288 : "drop table if exists buildids5_norm;\n"
289 : "drop table if exists buildids5_files;\n"
290 : "drop table if exists buildids5_buildids;\n"
291 : "drop table if exists buildids5_bolo;\n"
292 : "drop table if exists buildids5_rfolo;\n"
293 : "drop view if exists buildids5;\n"
294 : // buildids4: introduce rpmfile RFOLO
295 : "drop table if exists buildids4_norm;\n"
296 : "drop table if exists buildids4_files;\n"
297 : "drop table if exists buildids4_buildids;\n"
298 : "drop table if exists buildids4_bolo;\n"
299 : "drop table if exists buildids4_rfolo;\n"
300 : "drop view if exists buildids4;\n"
301 : // buildids3*: split out srcfile BOLO
302 : "drop table if exists buildids3_norm;\n"
303 : "drop table if exists buildids3_files;\n"
304 : "drop table if exists buildids3_buildids;\n"
305 : "drop table if exists buildids3_bolo;\n"
306 : "drop view if exists buildids3;\n"
307 : // buildids2: normalized buildid and filenames into interning tables;
308 : "drop table if exists buildids2_norm;\n"
309 : "drop table if exists buildids2_files;\n"
310 : "drop table if exists buildids2_buildids;\n"
311 : "drop view if exists buildids2;\n"
312 : // buildids1: made buildid and artifacttype NULLable, to represent cached-negative
313 : // lookups from sources, e.g. files or rpms that contain no buildid-indexable content
314 : "drop table if exists buildids1;\n"
315 : // buildids: original
316 : "drop table if exists buildids;\n"
317 : ;
318 :
319 : static const char DEBUGINFOD_SQLITE_CLEANUP_DDL[] =
320 : "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file
321 : ;
322 :
323 :
324 :
325 :
326 : /* Name and version of program. */
327 : /* ARGP_PROGRAM_VERSION_HOOK_DEF = print_version; */ // not this simple for C++
328 :
329 : /* Bug report address. */
330 : ARGP_PROGRAM_BUG_ADDRESS_DEF = PACKAGE_BUGREPORT;
331 :
332 : /* Definitions of arguments for argp functions. */
333 : static const struct argp_option options[] =
334 : {
335 : { NULL, 0, NULL, 0, "Scanners:", 1 },
336 : { "scan-file-dir", 'F', NULL, 0, "Enable ELF/DWARF file scanning.", 0 },
337 : { "scan-rpm-dir", 'R', NULL, 0, "Enable RPM scanning.", 0 },
338 : { "scan-deb-dir", 'U', NULL, 0, "Enable DEB scanning.", 0 },
339 : { "scan-archive", 'Z', "EXT=CMD", 0, "Enable arbitrary archive scanning.", 0 },
340 : // "source-oci-imageregistry" ...
341 :
342 : { NULL, 0, NULL, 0, "Options:", 2 },
343 : { "logical", 'L', NULL, 0, "Follow symlinks, default=ignore.", 0 },
344 : { "rescan-time", 't', "SECONDS", 0, "Number of seconds to wait between rescans, 0=disable.", 0 },
345 : { "groom-time", 'g', "SECONDS", 0, "Number of seconds to wait between database grooming, 0=disable.", 0 },
346 : { "maxigroom", 'G', NULL, 0, "Run a complete database groom/shrink pass at startup.", 0 },
347 : { "concurrency", 'c', "NUM", 0, "Limit scanning thread concurrency to NUM.", 0 },
348 : { "include", 'I', "REGEX", 0, "Include files matching REGEX, default=all.", 0 },
349 : { "exclude", 'X', "REGEX", 0, "Exclude files matching REGEX, default=none.", 0 },
350 : { "port", 'p', "NUM", 0, "HTTP port to listen on, default 8002.", 0 },
351 : { "database", 'd', "FILE", 0, "Path to sqlite database.", 0 },
352 : { "ddl", 'D', "SQL", 0, "Apply extra sqlite ddl/pragma to connection.", 0 },
353 : { "verbose", 'v', NULL, 0, "Increase verbosity.", 0 },
354 : #define ARGP_KEY_FDCACHE_FDS 0x1001
355 : { "fdcache-fds", ARGP_KEY_FDCACHE_FDS, "NUM", 0, "Maximum number of archive files to keep in fdcache.", 0 },
356 : #define ARGP_KEY_FDCACHE_MBS 0x1002
357 : { "fdcache-mbs", ARGP_KEY_FDCACHE_MBS, "MB", 0, "Maximum total size of archive file fdcache.", 0 },
358 : #define ARGP_KEY_FDCACHE_PREFETCH 0x1003
359 : { "fdcache-prefetch", ARGP_KEY_FDCACHE_PREFETCH, "NUM", 0, "Number of archive files to prefetch into fdcache.", 0 },
360 : { NULL, 0, NULL, 0, NULL, 0 }
361 : };
362 :
363 : /* Short description of program. */
364 : static const char doc[] = "Serve debuginfo-related content across HTTP from files under PATHs.";
365 :
366 : /* Strings for arguments in help texts. */
367 : static const char args_doc[] = "[PATH ...]";
368 :
369 : /* Prototype for option handler. */
370 : static error_t parse_opt (int key, char *arg, struct argp_state *state);
371 :
372 : /* Data structure to communicate with argp functions. */
373 : static struct argp argp =
374 : {
375 : options, parse_opt, args_doc, doc, NULL, NULL, NULL
376 : };
377 :
378 :
379 : static string db_path;
380 : static sqlite3 *db; // single connection, serialized across all our threads!
381 : static unsigned verbose;
382 : static volatile sig_atomic_t interrupted = 0;
383 : static volatile sig_atomic_t sigusr1 = 0;
384 : static volatile sig_atomic_t sigusr2 = 0;
385 : static unsigned http_port = 8002;
386 : static unsigned rescan_s = 300;
387 : static unsigned groom_s = 86400;
388 : static bool maxigroom = false;
389 : static unsigned concurrency = std::thread::hardware_concurrency() ?: 1;
390 : static set<string> source_paths;
391 : static bool scan_files = false;
392 : static map<string,string> scan_archives;
393 : static vector<string> extra_ddl;
394 : static regex_t file_include_regex;
395 : static regex_t file_exclude_regex;
396 : static bool traverse_logical;
397 : static long fdcache_fds;
398 : static long fdcache_mbs;
399 : static long fdcache_prefetch;
400 : static string tmpdir;
401 :
402 : static void set_metric(const string& key, int64_t value);
403 : // static void inc_metric(const string& key);
404 : static void set_metric(const string& metric,
405 : const string& lname, const string& lvalue,
406 : int64_t value);
407 : static void inc_metric(const string& metric,
408 : const string& lname, const string& lvalue);
409 : static void add_metric(const string& metric,
410 : const string& lname, const string& lvalue,
411 : int64_t value);
412 : // static void add_metric(const string& metric, int64_t value);
413 :
414 : /* Handle program arguments. */
415 : static error_t
416 41 : parse_opt (int key, char *arg,
417 : struct argp_state *state __attribute__ ((unused)))
418 : {
419 : int rc;
420 41 : switch (key)
421 : {
422 1 : case 'v': verbose ++; break;
423 3 : case 'd': db_path = string(arg); break;
424 3 : case 'p': http_port = (unsigned) atoi(arg);
425 3 : if (http_port == 0 || http_port > 65535)
426 0 : argp_failure(state, 1, EINVAL, "port number");
427 3 : break;
428 4 : case 'F': scan_files = true; break;
429 1 : case 'R':
430 1 : scan_archives[".rpm"]="cat"; // libarchive groks rpm natively
431 1 : break;
432 2 : case 'U':
433 2 : if (access("/usr/bin/dpkg-deb", X_OK) == 0)
434 : {
435 2 : scan_archives[".deb"]="dpkg-deb --fsys-tarfile";
436 2 : scan_archives[".ddeb"]="dpkg-deb --fsys-tarfile";
437 : }
438 : else
439 : {
440 0 : scan_archives[".deb"]="(bsdtar -O -x -f - data.tar.xz)<";
441 0 : scan_archives[".ddeb"]="(bsdtar -O -x -f - data.tar.xz)<";
442 : }
443 : // .udeb too?
444 2 : break;
445 2 : case 'Z':
446 : {
447 2 : char* extension = strchr(arg, '=');
448 2 : if (arg[0] == '\0')
449 0 : argp_failure(state, 1, EINVAL, "missing EXT");
450 2 : else if (extension)
451 1 : scan_archives[string(arg, (extension-arg))]=string(extension+1);
452 : else
453 1 : scan_archives[string(arg)]=string("cat");
454 : }
455 2 : break;
456 2 : case 'L':
457 2 : traverse_logical = true;
458 2 : break;
459 0 : case 'D': extra_ddl.push_back(string(arg)); break;
460 1 : case 't':
461 1 : rescan_s = (unsigned) atoi(arg);
462 1 : break;
463 1 : case 'g':
464 1 : groom_s = (unsigned) atoi(arg);
465 1 : break;
466 0 : case 'G':
467 0 : maxigroom = true;
468 0 : break;
469 0 : case 'c':
470 0 : concurrency = (unsigned) atoi(arg);
471 0 : if (concurrency < 1) concurrency = 1;
472 0 : break;
473 0 : case 'I':
474 : // NB: no problem with unconditional free here - an earlier failed regcomp would exit program
475 0 : regfree (&file_include_regex);
476 0 : rc = regcomp (&file_include_regex, arg, REG_EXTENDED|REG_NOSUB);
477 0 : if (rc != 0)
478 0 : argp_failure(state, 1, EINVAL, "regular expession");
479 0 : break;
480 0 : case 'X':
481 0 : regfree (&file_exclude_regex);
482 0 : rc = regcomp (&file_exclude_regex, arg, REG_EXTENDED|REG_NOSUB);
483 0 : if (rc != 0)
484 0 : argp_failure(state, 1, EINVAL, "regular expession");
485 0 : break;
486 1 : case ARGP_KEY_FDCACHE_FDS:
487 1 : fdcache_fds = atol (arg);
488 1 : break;
489 1 : case ARGP_KEY_FDCACHE_MBS:
490 1 : fdcache_mbs = atol (arg);
491 1 : break;
492 0 : case ARGP_KEY_FDCACHE_PREFETCH:
493 0 : fdcache_prefetch = atol (arg);
494 0 : break;
495 6 : case ARGP_KEY_ARG:
496 6 : source_paths.insert(string(arg));
497 6 : break;
498 : // case 'h': argp_state_help (state, stderr, ARGP_HELP_LONG|ARGP_HELP_EXIT_OK);
499 13 : default: return ARGP_ERR_UNKNOWN;
500 : }
501 :
502 28 : return 0;
503 : }
504 :
505 :
506 : ////////////////////////////////////////////////////////////////////////
507 :
508 :
509 : // represent errors that may get reported to an ostream and/or a libmicrohttpd connection
510 :
511 : struct reportable_exception
512 : {
513 : int code;
514 : string message;
515 :
516 6 : reportable_exception(int c, const string& m): code(c), message(m) {}
517 2 : reportable_exception(const string& m): code(503), message(m) {}
518 : reportable_exception(): code(503), message() {}
519 :
520 : void report(ostream& o) const; // defined under obatched() class below
521 :
522 8 : int mhd_send_response(MHD_Connection* c) const {
523 8 : MHD_Response* r = MHD_create_response_from_buffer (message.size(),
524 8 : (void*) message.c_str(),
525 : MHD_RESPMEM_MUST_COPY);
526 8 : MHD_add_response_header (r, "Content-Type", "text/plain");
527 8 : int rc = MHD_queue_response (c, code, r);
528 8 : MHD_destroy_response (r);
529 8 : return rc;
530 : }
531 : };
532 :
533 :
534 : struct sqlite_exception: public reportable_exception
535 : {
536 0 : sqlite_exception(int rc, const string& msg):
537 0 : reportable_exception(string("sqlite3 error: ") + msg + ": " + string(sqlite3_errstr(rc) ?: "?")) {}
538 : };
539 :
540 : struct libc_exception: public reportable_exception
541 : {
542 0 : libc_exception(int rc, const string& msg):
543 0 : reportable_exception(string("libc error: ") + msg + ": " + string(strerror(rc) ?: "?")) {}
544 : };
545 :
546 :
547 : struct archive_exception: public reportable_exception
548 : {
549 0 : archive_exception(const string& msg):
550 0 : reportable_exception(string("libarchive error: ") + msg) {}
551 0 : archive_exception(struct archive* a, const string& msg):
552 0 : reportable_exception(string("libarchive error: ") + msg + ": " + string(archive_error_string(a) ?: "?")) {}
553 : };
554 :
555 :
556 : struct elfutils_exception: public reportable_exception
557 : {
558 0 : elfutils_exception(int rc, const string& msg):
559 0 : reportable_exception(string("elfutils error: ") + msg + ": " + string(elf_errmsg(rc) ?: "?")) {}
560 : };
561 :
562 :
563 : ////////////////////////////////////////////////////////////////////////
564 :
565 : template <typename Payload>
566 : class workq
567 : {
568 : set<Payload> q; // eliminate duplicates
569 : mutex mtx;
570 : condition_variable cv;
571 : bool dead;
572 : unsigned idlers;
573 :
574 : public:
575 3 : workq() { dead = false; idlers = 0; }
576 3 : ~workq() {}
577 :
578 54 : void push_back(const Payload& p)
579 : {
580 108 : unique_lock<mutex> lock(mtx);
581 54 : q.insert (p);
582 54 : set_metric("thread_work_pending","role","scan", q.size());
583 54 : cv.notify_all();
584 54 : }
585 :
586 : // kill this workqueue, wake up all idlers / scanners
587 3 : void nuke() {
588 6 : unique_lock<mutex> lock(mtx);
589 : // optional: q.clear();
590 3 : dead = true;
591 3 : cv.notify_all();
592 3 : }
593 :
594 : // block this scanner thread until there is work to do and no active
595 66 : bool wait_front (Payload& p)
596 : {
597 132 : unique_lock<mutex> lock(mtx);
598 248 : while (!dead && (q.size() == 0 || idlers > 0))
599 182 : cv.wait(lock);
600 66 : if (dead)
601 12 : return false;
602 : else
603 : {
604 54 : p = * q.begin();
605 54 : q.erase (q.begin());
606 54 : set_metric("thread_work_pending","role","scan", q.size());
607 54 : if (q.size() == 0)
608 37 : cv.notify_all(); // maybe wake up waiting idlers
609 54 : return true;
610 : }
611 : }
612 :
613 : // block this idler thread until there is no work to do
614 26 : void wait_idle ()
615 : {
616 26 : unique_lock<mutex> lock(mtx);
617 26 : cv.notify_all(); // maybe wake up waiting scanners
618 27 : while (!dead && (q.size() != 0))
619 1 : cv.wait(lock);
620 26 : idlers ++;
621 26 : }
622 :
623 23 : void done_idle ()
624 : {
625 46 : unique_lock<mutex> lock(mtx);
626 23 : idlers --;
627 23 : cv.notify_all(); // maybe wake up waiting scanners, but probably not (shutting down)
628 23 : }
629 : };
630 :
631 : typedef struct stat stat_t;
632 : typedef pair<string,stat_t> scan_payload;
633 88 : inline bool operator< (const scan_payload& a, const scan_payload& b)
634 : {
635 88 : return a.first < b.first; // don't bother compare the stat fields
636 : }
637 : static workq<scan_payload> scanq; // just a single one
638 : // producer & idler: thread_main_fts_source_paths()
639 : // consumer: thread_main_scanner()
640 : // idler: thread_main_groom()
641 :
642 :
643 :
644 : ////////////////////////////////////////////////////////////////////////
645 :
646 :
647 : // Print a standard timestamp.
648 : static ostream&
649 186 : timestamp (ostream &o)
650 : {
651 : char datebuf[80];
652 186 : char *now2 = NULL;
653 186 : time_t now_t = time(NULL);
654 186 : struct tm *now = gmtime (&now_t);
655 186 : if (now)
656 : {
657 186 : (void) strftime (datebuf, sizeof (datebuf), "%c", now);
658 186 : now2 = datebuf;
659 : }
660 :
661 : return o << "[" << (now2 ? now2 : "") << "] "
662 186 : << "(" << getpid () << "/" << tid() << "): ";
663 : }
664 :
665 :
666 : // A little class that impersonates an ostream to the extent that it can
667 : // take << streaming operations. It batches up the bits into an internal
668 : // stringstream until it is destroyed; then flushes to the original ostream.
669 : // It adds a timestamp
670 : class obatched
671 : {
672 : private:
673 : ostream& o;
674 : stringstream stro;
675 : static mutex lock;
676 : public:
677 186 : obatched(ostream& oo, bool timestamp_p = true): o(oo)
678 : {
679 186 : if (timestamp_p)
680 186 : timestamp(stro);
681 186 : }
682 186 : ~obatched()
683 186 : {
684 372 : unique_lock<mutex> do_not_cross_the_streams(obatched::lock);
685 186 : o << stro.str();
686 186 : o.flush();
687 186 : }
688 : operator ostream& () { return stro; }
689 186 : template <typename T> ostream& operator << (const T& t) { stro << t; return stro; }
690 : };
691 : mutex obatched::lock; // just the one, since cout/cerr iostreams are not thread-safe
692 :
693 :
694 8 : void reportable_exception::report(ostream& o) const {
695 8 : obatched(o) << message << endl;
696 8 : }
697 :
698 :
699 : ////////////////////////////////////////////////////////////////////////
700 :
701 :
702 : // RAII style sqlite prepared-statement holder that matches { } block lifetime
703 :
704 : struct sqlite_ps
705 : {
706 : private:
707 : sqlite3* db;
708 : const string nickname;
709 : const string sql;
710 : sqlite3_stmt *pp;
711 :
712 : sqlite_ps(const sqlite_ps&); // make uncopyable
713 : sqlite_ps& operator=(const sqlite_ps &); // make unassignable
714 :
715 : public:
716 236 : sqlite_ps (sqlite3* d, const string& n, const string& s): db(d), nickname(n), sql(s) {
717 236 : if (verbose > 4)
718 0 : obatched(clog) << nickname << " prep " << sql << endl;
719 236 : int rc = sqlite3_prepare_v2 (db, sql.c_str(), -1 /* to \0 */, & this->pp, NULL);
720 236 : if (rc != SQLITE_OK)
721 0 : throw sqlite_exception(rc, "prepare " + sql);
722 236 : }
723 :
724 2084 : sqlite_ps& reset()
725 : {
726 2084 : sqlite3_reset(this->pp);
727 2084 : return *this;
728 : }
729 :
730 3191 : sqlite_ps& bind(int parameter, const string& str)
731 : {
732 3191 : if (verbose > 4)
733 0 : obatched(clog) << nickname << " bind " << parameter << "=" << str << endl;
734 3191 : int rc = sqlite3_bind_text (this->pp, parameter, str.c_str(), -1, SQLITE_TRANSIENT);
735 3191 : if (rc != SQLITE_OK)
736 0 : throw sqlite_exception(rc, "sqlite3 bind");
737 3191 : return *this;
738 : }
739 :
740 836 : sqlite_ps& bind(int parameter, int64_t value)
741 : {
742 836 : if (verbose > 4)
743 0 : obatched(clog) << nickname << " bind " << parameter << "=" << value << endl;
744 836 : int rc = sqlite3_bind_int64 (this->pp, parameter, value);
745 835 : if (rc != SQLITE_OK)
746 0 : throw sqlite_exception(rc, "sqlite3 bind");
747 836 : return *this;
748 : }
749 :
750 : sqlite_ps& bind(int parameter)
751 : {
752 : if (verbose > 4)
753 : obatched(clog) << nickname << " bind " << parameter << "=" << "NULL" << endl;
754 : int rc = sqlite3_bind_null (this->pp, parameter);
755 : if (rc != SQLITE_OK)
756 : throw sqlite_exception(rc, "sqlite3 bind");
757 : return *this;
758 : }
759 :
760 :
761 1860 : void step_ok_done() {
762 1860 : int rc = sqlite3_step (this->pp);
763 1860 : if (verbose > 4)
764 0 : obatched(clog) << nickname << " step-ok-done(" << sqlite3_errstr(rc) << ") " << sql << endl;
765 1860 : if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)
766 0 : throw sqlite_exception(rc, "sqlite3 step");
767 1860 : (void) sqlite3_reset (this->pp);
768 1860 : }
769 :
770 :
771 170 : int step() {
772 170 : int rc = sqlite3_step (this->pp);
773 170 : if (verbose > 4)
774 0 : obatched(clog) << nickname << " step(" << sqlite3_errstr(rc) << ") " << sql << endl;
775 170 : return rc;
776 : }
777 :
778 :
779 :
780 235 : ~sqlite_ps () { sqlite3_finalize (this->pp); }
781 460 : operator sqlite3_stmt* () { return this->pp; }
782 : };
783 :
784 :
785 : ////////////////////////////////////////////////////////////////////////
786 :
787 : // RAII style templated autocloser
788 :
789 : template <class Payload, class Ignore>
790 : struct defer_dtor
791 : {
792 : public:
793 : typedef Ignore (*dtor_fn) (Payload);
794 :
795 : private:
796 : Payload p;
797 : dtor_fn fn;
798 :
799 : public:
800 161 : defer_dtor(Payload _p, dtor_fn _fn): p(_p), fn(_fn) {}
801 163 : ~defer_dtor() { (void) (*fn)(p); }
802 :
803 : private:
804 : defer_dtor(const defer_dtor<Payload,Ignore>&); // make uncopyable
805 : defer_dtor& operator=(const defer_dtor<Payload,Ignore> &); // make unassignable
806 : };
807 :
808 :
809 :
810 : ////////////////////////////////////////////////////////////////////////
811 :
812 :
813 : static string
814 168 : header_censor(const string& str)
815 : {
816 168 : string y;
817 2155 : for (auto&& x : str)
818 : {
819 1987 : if (isalnum(x) || x == '/' || x == '.' || x == ',' || x == '_' || x == ':')
820 1986 : y += x;
821 : }
822 168 : return y;
823 : }
824 :
825 :
826 : static string
827 84 : conninfo (struct MHD_Connection * conn)
828 : {
829 : char hostname[256]; // RFC1035
830 : char servname[256];
831 84 : int sts = -1;
832 :
833 84 : if (conn == 0)
834 0 : return "internal";
835 :
836 : /* Look up client address data. */
837 84 : const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
838 : MHD_CONNECTION_INFO_CLIENT_ADDRESS);
839 84 : struct sockaddr *so = u ? u->client_addr : 0;
840 :
841 84 : if (so && so->sa_family == AF_INET) {
842 84 : sts = getnameinfo (so, sizeof (struct sockaddr_in), hostname, sizeof (hostname), servname,
843 : sizeof (servname), NI_NUMERICHOST | NI_NUMERICSERV);
844 0 : } else if (so && so->sa_family == AF_INET6) {
845 0 : sts = getnameinfo (so, sizeof (struct sockaddr_in6), hostname, sizeof (hostname),
846 : servname, sizeof (servname), NI_NUMERICHOST | NI_NUMERICSERV);
847 : }
848 84 : if (sts != 0) {
849 0 : hostname[0] = servname[0] = '\0';
850 : }
851 :
852 : // extract headers relevant to administration
853 84 : const char* user_agent = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";
854 84 : const char* x_forwarded_for = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";
855 : // NB: these are untrustworthy, beware if machine-processing log files
856 :
857 168 : return string(hostname) + string(":") + string(servname) +
858 336 : string(" UA:") + header_censor(string(user_agent)) +
859 252 : string(" XFF:") + header_censor(string(x_forwarded_for));
860 : }
861 :
862 :
863 :
864 : ////////////////////////////////////////////////////////////////////////
865 :
866 :
867 : static void
868 38 : add_mhd_last_modified (struct MHD_Response *resp, time_t mtime)
869 : {
870 38 : struct tm *now = gmtime (&mtime);
871 38 : if (now != NULL)
872 : {
873 : char datebuf[80];
874 38 : size_t rc = strftime (datebuf, sizeof (datebuf), "%a, %d %b %Y %T GMT", now);
875 38 : if (rc > 0 && rc < sizeof (datebuf))
876 38 : (void) MHD_add_response_header (resp, "Last-Modified", datebuf);
877 : }
878 :
879 38 : (void) MHD_add_response_header (resp, "Cache-Control", "public");
880 38 : }
881 :
882 :
883 :
884 : static struct MHD_Response*
885 14 : handle_buildid_f_match (int64_t b_mtime,
886 : const string& b_source0,
887 : int *result_fd)
888 : {
889 14 : int fd = open(b_source0.c_str(), O_RDONLY);
890 14 : if (fd < 0)
891 : {
892 0 : if (verbose)
893 0 : obatched(clog) << "cannot open " << b_source0 << endl;
894 : // if still missing, a periodic groom pass will delete this buildid record
895 0 : return 0;
896 : }
897 :
898 : // NB: use manual close(2) in error case instead of defer_dtor, because
899 : // in the normal case, we want to hand the fd over to libmicrohttpd for
900 : // file transfer.
901 :
902 : struct stat s;
903 14 : int rc = fstat(fd, &s);
904 14 : if (rc < 0)
905 : {
906 0 : if (verbose)
907 0 : clog << "cannot fstat " << b_source0 << endl;
908 0 : close(fd);
909 0 : return 0;
910 : }
911 :
912 14 : if ((int64_t) s.st_mtime != b_mtime)
913 : {
914 0 : if (verbose)
915 0 : obatched(clog) << "mtime mismatch for " << b_source0 << endl;
916 0 : close(fd);
917 0 : return 0;
918 : }
919 :
920 14 : inc_metric ("http_responses_total","result","file");
921 14 : struct MHD_Response* r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd);
922 14 : if (r == 0)
923 : {
924 0 : if (verbose)
925 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
926 0 : close(fd);
927 : }
928 : else
929 : {
930 14 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
931 14 : add_mhd_last_modified (r, s.st_mtime);
932 14 : if (verbose > 1)
933 0 : obatched(clog) << "serving file " << b_source0 << endl;
934 : /* libmicrohttpd will close it. */
935 14 : if (result_fd)
936 14 : *result_fd = fd;
937 : }
938 :
939 14 : return r;
940 : }
941 :
942 :
943 : // quote all questionable characters of str for safe passage through a sh -c expansion.
944 : static string
945 2 : shell_escape(const string& str)
946 : {
947 2 : string y;
948 196 : for (auto&& x : str)
949 : {
950 194 : if (! isalnum(x) && x != '/')
951 22 : y += "\\";
952 194 : y += x;
953 : }
954 2 : return y;
955 : }
956 :
957 :
958 : // PR25548: Perform POSIX / RFC3986 style path canonicalization on the input string.
959 : //
960 : // Namely:
961 : // // -> /
962 : // /foo/../ -> /
963 : // /./ -> /
964 : //
965 : // This mapping is done on dwarf-side source path names, which may
966 : // include these constructs, so we can deal with debuginfod clients
967 : // that accidentally canonicalize the paths.
968 : //
969 : // realpath(3) is close but not quite right, because it also resolves
970 : // symbolic links. Symlinks at the debuginfod server have nothing to
971 : // do with the build-time symlinks, thus they must not be considered.
972 : //
973 : // see also curl Curl_dedotdotify() aka RFC3986, which we mostly follow here
974 : // see also libc __realpath()
975 : // see also llvm llvm::sys::path::remove_dots()
976 : static string
977 515 : canon_pathname (const string& input)
978 : {
979 1030 : string i = input; // 5.2.4 (1)
980 515 : string o;
981 :
982 3250 : while (i.size() != 0)
983 : {
984 : // 5.2.4 (2) A
985 2735 : if (i.substr(0,3) == "../")
986 0 : i = i.substr(3);
987 2735 : else if(i.substr(0,2) == "./")
988 0 : i = i.substr(2);
989 :
990 : // 5.2.4 (2) B
991 2735 : else if (i.substr(0,3) == "/./")
992 51 : i = i.substr(2);
993 2684 : else if (i == "/.")
994 0 : i = ""; // no need to handle "/." complete-path-segment case; we're dealing with file names
995 :
996 : // 5.2.4 (2) C
997 2684 : else if (i.substr(0,4) == "/../") {
998 73 : i = i.substr(3);
999 73 : string::size_type sl = o.rfind("/");
1000 73 : if (sl != string::npos)
1001 73 : o = o.substr(0, sl);
1002 : else
1003 0 : o = "";
1004 2611 : } else if (i == "/..")
1005 0 : i = ""; // no need to handle "/.." complete-path-segment case; we're dealing with file names
1006 :
1007 : // 5.2.4 (2) D
1008 : // no need to handle these cases; we're dealing with file names
1009 2611 : else if (i == ".")
1010 0 : i = "";
1011 2611 : else if (i == "..")
1012 0 : i = "";
1013 :
1014 : // POSIX special: map // to /
1015 2611 : else if (i.substr(0,2) == "//")
1016 4 : i = i.substr(1);
1017 :
1018 : // 5.2.4 (2) E
1019 : else {
1020 2607 : string::size_type next_slash = i.find("/", (i[0]=='/' ? 1 : 0)); // skip first slash
1021 2607 : o += i.substr(0, next_slash);
1022 2607 : if (next_slash == string::npos)
1023 515 : i = "";
1024 : else
1025 2092 : i = i.substr(next_slash);
1026 : }
1027 : }
1028 :
1029 1030 : return o;
1030 : }
1031 :
1032 :
1033 :
1034 : // A map-like class that owns a cache of file descriptors (indexed by
1035 : // file / content names).
1036 : //
1037 : // If only it could use fd's instead of file names ... but we can't
1038 : // dup(2) to create independent descriptors for the same unlinked
1039 : // files, so would have to use some goofy linux /proc/self/fd/%d
1040 : // hack such as the following
1041 :
1042 : #if 0
1043 : int superdup(int fd)
1044 : {
1045 : #ifdef __linux__
1046 : char *fdpath = NULL;
1047 : int rc = asprintf(& fdpath, "/proc/self/fd/%d", fd);
1048 : int newfd;
1049 : if (rc >= 0)
1050 : newfd = open(fdpath, O_RDONLY);
1051 : else
1052 : newfd = -1;
1053 : free (fdpath);
1054 : return newfd;
1055 : #else
1056 : return -1;
1057 : #endif
1058 : }
1059 : #endif
1060 :
1061 : class libarchive_fdcache
1062 : {
1063 : private:
1064 : mutex fdcache_lock;
1065 :
1066 : struct fdcache_entry
1067 : {
1068 : string archive;
1069 : string entry;
1070 : string fd;
1071 : double fd_size_mb; // slightly rounded up megabytes
1072 : };
1073 : deque<fdcache_entry> lru; // @head: most recently used
1074 : long max_fds;
1075 : long max_mbs;
1076 :
1077 : public:
1078 27 : void intern(const string& a, const string& b, string fd, off_t sz, bool front_p)
1079 : {
1080 : {
1081 54 : unique_lock<mutex> lock(fdcache_lock);
1082 58 : for (auto i = lru.begin(); i < lru.end(); i++) // nuke preexisting copy
1083 : {
1084 31 : if (i->archive == a && i->entry == b)
1085 : {
1086 0 : unlink (i->fd.c_str());
1087 0 : lru.erase(i);
1088 0 : break; // must not continue iterating
1089 : }
1090 : }
1091 27 : double mb = (sz+65535)/1048576.0; // round up to 64K block
1092 54 : fdcache_entry n = { a, b, fd, mb };
1093 27 : if (front_p)
1094 17 : lru.push_front(n);
1095 : else
1096 10 : lru.push_back(n);
1097 27 : if (verbose > 3)
1098 0 : obatched(clog) << "fdcache interned a=" << a << " b=" << b
1099 0 : << " fd=" << fd << " mb=" << mb << " front=" << front_p << endl;
1100 : }
1101 :
1102 : // NB: we age the cache at lookup time too
1103 27 : if (front_p)
1104 17 : this->limit(max_fds, max_mbs); // age cache if required
1105 27 : }
1106 :
1107 23 : int lookup(const string& a, const string& b)
1108 : {
1109 23 : int fd = -1;
1110 : {
1111 46 : unique_lock<mutex> lock(fdcache_lock);
1112 46 : for (auto i = lru.begin(); i < lru.end(); i++)
1113 : {
1114 29 : if (i->archive == a && i->entry == b)
1115 : { // found it; move it to head of lru
1116 12 : fdcache_entry n = *i;
1117 6 : lru.erase(i); // invalidates i, so no more iteration!
1118 6 : lru.push_front(n);
1119 :
1120 6 : fd = open(n.fd.c_str(), O_RDONLY); // NB: no problem if dup() fails; looks like cache miss
1121 6 : break;
1122 : }
1123 : }
1124 : }
1125 :
1126 23 : if (fd >= 0)
1127 6 : this->limit(max_fds, max_mbs); // age cache if required
1128 :
1129 23 : return fd;
1130 : }
1131 :
1132 27 : int probe(const string& a, const string& b) // just a cache residency check - don't modify LRU state, don't open
1133 : {
1134 54 : unique_lock<mutex> lock(fdcache_lock);
1135 58 : for (auto i = lru.begin(); i < lru.end(); i++)
1136 : {
1137 31 : if (i->archive == a && i->entry == b)
1138 0 : return true;
1139 : }
1140 27 : return false;
1141 : }
1142 :
1143 0 : void clear(const string& a, const string& b)
1144 : {
1145 0 : unique_lock<mutex> lock(fdcache_lock);
1146 0 : for (auto i = lru.begin(); i < lru.end(); i++)
1147 : {
1148 0 : if (i->archive == a && i->entry == b)
1149 : { // found it; move it to head of lru
1150 0 : fdcache_entry n = *i;
1151 0 : lru.erase(i); // invalidates i, so no more iteration!
1152 0 : unlink (n.fd.c_str());
1153 0 : return;
1154 : }
1155 : }
1156 : }
1157 :
1158 37 : void limit(long maxfds, long maxmbs)
1159 : {
1160 37 : if (verbose > 3 && (this->max_fds != maxfds || this->max_mbs != maxmbs))
1161 0 : obatched(clog) << "fdcache limited to maxfds=" << maxfds << " maxmbs=" << maxmbs << endl;
1162 :
1163 74 : unique_lock<mutex> lock(fdcache_lock);
1164 37 : this->max_fds = maxfds;
1165 37 : this->max_mbs = maxmbs;
1166 :
1167 37 : long total_fd = 0;
1168 37 : double total_mb = 0.0;
1169 60 : for (auto i = lru.begin(); i < lru.end(); i++)
1170 : {
1171 : // accumulate totals from most recently used one going backward
1172 46 : total_fd ++;
1173 46 : total_mb += i->fd_size_mb;
1174 46 : if (total_fd > max_fds || total_mb > max_mbs)
1175 : {
1176 : // found the cut here point!
1177 :
1178 50 : for (auto j = i; j < lru.end(); j++) // close all the fds from here on in
1179 : {
1180 27 : if (verbose > 3)
1181 0 : obatched(clog) << "fdcache evicted a=" << j->archive << " b=" << j->entry
1182 0 : << " fd=" << j->fd << " mb=" << j->fd_size_mb << endl;
1183 27 : unlink (j->fd.c_str());
1184 : }
1185 :
1186 23 : lru.erase(i, lru.end()); // erase the nodes generally
1187 23 : break;
1188 : }
1189 :
1190 : }
1191 37 : }
1192 :
1193 3 : ~libarchive_fdcache()
1194 3 : {
1195 3 : limit(0, 0);
1196 3 : }
1197 : };
1198 : static libarchive_fdcache fdcache;
1199 :
1200 :
1201 : // For security/portability reasons, many distro-package archives have
1202 : // a "./" in front of path names; others have nothing, others have
1203 : // "/". Canonicalize them all to a single leading "/", with the
1204 : // assumption that this matches the dwarf-derived file names too.
1205 70 : string canonicalized_archive_entry_pathname(struct archive_entry *e)
1206 : {
1207 140 : string fn = archive_entry_pathname(e);
1208 70 : if (fn.size() == 0)
1209 0 : return fn;
1210 69 : if (fn[0] == '/')
1211 0 : return fn;
1212 69 : if (fn[0] == '.')
1213 58 : return fn.substr(1);
1214 : else
1215 11 : return string("/")+fn;
1216 : }
1217 :
1218 :
1219 :
1220 : static struct MHD_Response*
1221 23 : handle_buildid_r_match (int64_t b_mtime,
1222 : const string& b_source0,
1223 : const string& b_source1,
1224 : int *result_fd)
1225 : {
1226 : struct stat fs;
1227 23 : int rc = stat (b_source0.c_str(), &fs);
1228 23 : if (rc != 0)
1229 0 : throw libc_exception (errno, string("stat ") + b_source0);
1230 :
1231 23 : if ((int64_t) fs.st_mtime != b_mtime)
1232 : {
1233 0 : if (verbose)
1234 0 : obatched(clog) << "mtime mismatch for " << b_source0 << endl;
1235 0 : return 0;
1236 : }
1237 :
1238 : // check for a match in the fdcache first
1239 23 : int fd = fdcache.lookup(b_source0, b_source1);
1240 23 : while (fd >= 0) // got one!; NB: this is really an if() with a possible branch out to the end
1241 : {
1242 6 : rc = fstat(fd, &fs);
1243 6 : if (rc < 0) // disappeared?
1244 : {
1245 0 : if (verbose)
1246 0 : obatched(clog) << "cannot fstat fdcache " << b_source0 << endl;
1247 0 : close(fd);
1248 0 : fdcache.clear(b_source0, b_source1);
1249 0 : break; // branch out of if "loop", to try new libarchive fetch attempt
1250 : }
1251 :
1252 6 : struct MHD_Response* r = MHD_create_response_from_fd (fs.st_size, fd);
1253 6 : if (r == 0)
1254 : {
1255 0 : if (verbose)
1256 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
1257 0 : close(fd);
1258 0 : break; // branch out of if "loop", to try new libarchive fetch attempt
1259 : }
1260 :
1261 6 : inc_metric ("http_responses_total","result","archive fdcache");
1262 :
1263 6 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
1264 6 : add_mhd_last_modified (r, fs.st_mtime);
1265 6 : if (verbose > 1)
1266 0 : obatched(clog) << "serving fdcache archive " << b_source0 << " file " << b_source1 << endl;
1267 : /* libmicrohttpd will close it. */
1268 6 : if (result_fd)
1269 6 : *result_fd = fd;
1270 6 : return r;
1271 : // NB: see, we never go around the 'loop' more than once
1272 : }
1273 :
1274 : // no match ... grumble, must process the archive
1275 34 : string archive_decoder = "/dev/null";
1276 34 : string archive_extension = "";
1277 68 : for (auto&& arch : scan_archives)
1278 51 : if (string_endswith(b_source0, arch.first))
1279 : {
1280 17 : archive_extension = arch.first;
1281 17 : archive_decoder = arch.second;
1282 : }
1283 : FILE* fp;
1284 : defer_dtor<FILE*,int>::dtor_fn dfn;
1285 17 : if (archive_decoder != "cat")
1286 : {
1287 3 : string popen_cmd = archive_decoder + " " + shell_escape(b_source0);
1288 1 : fp = popen (popen_cmd.c_str(), "r"); // "e" O_CLOEXEC?
1289 1 : dfn = pclose;
1290 1 : if (fp == NULL)
1291 0 : throw libc_exception (errno, string("popen ") + popen_cmd);
1292 : }
1293 : else
1294 : {
1295 16 : fp = fopen (b_source0.c_str(), "r");
1296 16 : dfn = fclose;
1297 16 : if (fp == NULL)
1298 0 : throw libc_exception (errno, string("fopen ") + b_source0);
1299 : }
1300 34 : defer_dtor<FILE*,int> fp_closer (fp, dfn);
1301 :
1302 : struct archive *a;
1303 17 : a = archive_read_new();
1304 17 : if (a == NULL)
1305 0 : throw archive_exception("cannot create archive reader");
1306 17 : defer_dtor<struct archive*,int> archive_closer (a, archive_read_free);
1307 :
1308 17 : rc = archive_read_support_format_all(a);
1309 17 : if (rc != ARCHIVE_OK)
1310 0 : throw archive_exception(a, "cannot select all format");
1311 17 : rc = archive_read_support_filter_all(a);
1312 17 : if (rc != ARCHIVE_OK)
1313 0 : throw archive_exception(a, "cannot select all filters");
1314 :
1315 17 : rc = archive_read_open_FILE (a, fp);
1316 17 : if (rc != ARCHIVE_OK)
1317 0 : throw archive_exception(a, "cannot open archive from pipe");
1318 :
1319 : // archive traversal is in three stages, no, four stages:
1320 : // 1) skip entries whose names do not match the requested one
1321 : // 2) extract the matching entry name (set r = result)
1322 : // 3) extract some number of prefetched entries (just into fdcache)
1323 : // 4) abort any further processing
1324 17 : struct MHD_Response* r = 0; // will set in stage 2
1325 17 : unsigned prefetch_count = fdcache_prefetch; // will decrement in stage 3
1326 :
1327 162 : while(r == 0 || prefetch_count > 0) // stage 1, 2, or 3
1328 : {
1329 162 : if (interrupted)
1330 17 : break;
1331 :
1332 : struct archive_entry *e;
1333 162 : rc = archive_read_next_header (a, &e);
1334 162 : if (rc != ARCHIVE_OK)
1335 17 : break;
1336 :
1337 145 : if (! S_ISREG(archive_entry_mode (e))) // skip non-files completely
1338 145 : continue;
1339 :
1340 38 : string fn = canonicalized_archive_entry_pathname (e);
1341 38 : if ((r == 0) && (fn != b_source1)) // stage 1
1342 11 : continue;
1343 :
1344 27 : if (fdcache.probe (b_source0, fn)) // skip if already interned
1345 0 : continue;
1346 :
1347 : // extract this file to a temporary file
1348 27 : char* tmppath = NULL;
1349 27 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
1350 27 : if (rc < 0)
1351 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
1352 27 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
1353 27 : fd = mkstemp (tmppath);
1354 27 : if (fd < 0)
1355 0 : throw libc_exception (errno, "cannot create temporary file");
1356 : // NB: don't unlink (tmppath), as fdcache will take charge of it.
1357 :
1358 27 : rc = archive_read_data_into_fd (a, fd);
1359 27 : if (rc != ARCHIVE_OK) // e.g. ENOSPC!
1360 : {
1361 0 : close (fd);
1362 0 : unlink (tmppath);
1363 0 : throw archive_exception(a, "cannot extract file");
1364 : }
1365 :
1366 : // Set the mtime so the fdcache file mtimes, even prefetched ones,
1367 : // propagate to future webapi clients.
1368 : struct timeval tvs[2];
1369 27 : tvs[0].tv_sec = tvs[1].tv_sec = archive_entry_mtime(e);
1370 27 : tvs[0].tv_usec = tvs[1].tv_usec = 0;
1371 27 : (void) futimes (fd, tvs); /* best effort */
1372 :
1373 27 : if (r != 0) // stage 3
1374 : {
1375 : // NB: now we know we have a complete reusable file; make fdcache
1376 : // responsible for unlinking it later.
1377 10 : fdcache.intern(b_source0, fn,
1378 : tmppath, archive_entry_size(e),
1379 : false); // prefetched ones go to back of lru
1380 10 : prefetch_count --;
1381 10 : close (fd); // we're not saving this fd to make a mhd-response from!
1382 10 : continue;
1383 : }
1384 :
1385 : // NB: now we know we have a complete reusable file; make fdcache
1386 : // responsible for unlinking it later.
1387 17 : fdcache.intern(b_source0, b_source1,
1388 : tmppath, archive_entry_size(e),
1389 : true); // requested ones go to the front of lru
1390 :
1391 17 : inc_metric ("http_responses_total","result",archive_extension + " archive");
1392 17 : r = MHD_create_response_from_fd (archive_entry_size(e), fd);
1393 17 : if (r == 0)
1394 : {
1395 0 : if (verbose)
1396 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
1397 0 : close(fd);
1398 0 : break; // assume no chance of better luck around another iteration; no other copies of same file
1399 : }
1400 : else
1401 : {
1402 17 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
1403 17 : add_mhd_last_modified (r, archive_entry_mtime(e));
1404 17 : if (verbose > 1)
1405 0 : obatched(clog) << "serving archive " << b_source0 << " file " << b_source1 << endl;
1406 : /* libmicrohttpd will close it. */
1407 17 : if (result_fd)
1408 17 : *result_fd = fd;
1409 17 : continue;
1410 : }
1411 : }
1412 :
1413 : // XXX: rpm/file not found: delete this R entry?
1414 17 : return r;
1415 : }
1416 :
1417 :
1418 : static struct MHD_Response*
1419 37 : handle_buildid_match (int64_t b_mtime,
1420 : const string& b_stype,
1421 : const string& b_source0,
1422 : const string& b_source1,
1423 : int *result_fd)
1424 : {
1425 37 : if (b_stype == "F")
1426 14 : return handle_buildid_f_match(b_mtime, b_source0, result_fd);
1427 23 : else if (b_stype == "R")
1428 23 : return handle_buildid_r_match(b_mtime, b_source0, b_source1, result_fd);
1429 : else
1430 0 : return 0;
1431 : }
1432 :
1433 :
1434 : static int
1435 6 : debuginfod_find_progress (debuginfod_client *, long a, long b)
1436 : {
1437 6 : if (verbose > 4)
1438 0 : obatched(clog) << "federated debuginfod progress=" << a << "/" << b << endl;
1439 :
1440 6 : return interrupted;
1441 : }
1442 :
1443 :
1444 : static struct MHD_Response*
1445 44 : handle_buildid (MHD_Connection* conn,
1446 : const string& buildid /* unsafe */,
1447 : const string& artifacttype /* unsafe */,
1448 : const string& suffix /* unsafe */,
1449 : int *result_fd)
1450 : {
1451 : // validate artifacttype
1452 88 : string atype_code;
1453 44 : if (artifacttype == "debuginfo") atype_code = "D";
1454 22 : else if (artifacttype == "executable") atype_code = "E";
1455 10 : else if (artifacttype == "source") atype_code = "S";
1456 0 : else throw reportable_exception("invalid artifacttype");
1457 :
1458 44 : if (atype_code == "S" && suffix == "")
1459 0 : throw reportable_exception("invalid source suffix");
1460 :
1461 : // validate buildid
1462 44 : if ((buildid.size() < 2) || // not empty
1463 88 : (buildid.size() % 2) || // even number
1464 44 : (buildid.find_first_not_of("0123456789abcdef") != string::npos)) // pure tasty lowercase hex
1465 0 : throw reportable_exception("invalid buildid");
1466 :
1467 44 : if (verbose > 1)
1468 0 : obatched(clog) << "searching for buildid=" << buildid << " artifacttype=" << artifacttype
1469 0 : << " suffix=" << suffix << endl;
1470 :
1471 44 : sqlite_ps *pp = 0;
1472 :
1473 44 : if (atype_code == "D")
1474 : {
1475 44 : pp = new sqlite_ps (db, "mhd-query-d",
1476 : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_d where buildid = ? "
1477 22 : "order by mtime desc");
1478 22 : pp->reset();
1479 22 : pp->bind(1, buildid);
1480 : }
1481 22 : else if (atype_code == "E")
1482 : {
1483 24 : pp = new sqlite_ps (db, "mhd-query-e",
1484 : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_e where buildid = ? "
1485 12 : "order by mtime desc");
1486 12 : pp->reset();
1487 12 : pp->bind(1, buildid);
1488 : }
1489 10 : else if (atype_code == "S")
1490 : {
1491 : // PR25548
1492 : // Incoming source queries may come in with either dwarf-level OR canonicalized paths.
1493 : // We let the query pass with either one.
1494 :
1495 20 : pp = new sqlite_ps (db, "mhd-query-s",
1496 : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_s where buildid = ? and artifactsrc in (?,?) "
1497 10 : "order by sharedprefix(source0,source0ref) desc, mtime desc");
1498 10 : pp->reset();
1499 10 : pp->bind(1, buildid);
1500 10 : pp->bind(2, suffix);
1501 10 : pp->bind(3, canon_pathname(suffix));
1502 : }
1503 88 : unique_ptr<sqlite_ps> ps_closer(pp); // release pp if exception or return
1504 :
1505 : // consume all the rows
1506 : while (1)
1507 : {
1508 44 : int rc = pp->step();
1509 44 : if (rc == SQLITE_DONE) break;
1510 37 : if (rc != SQLITE_ROW)
1511 0 : throw sqlite_exception(rc, "step");
1512 :
1513 37 : int64_t b_mtime = sqlite3_column_int64 (*pp, 0);
1514 37 : string b_stype = string((const char*) sqlite3_column_text (*pp, 1) ?: ""); /* by DDL may not be NULL */
1515 37 : string b_source0 = string((const char*) sqlite3_column_text (*pp, 2) ?: ""); /* may be NULL */
1516 37 : string b_source1 = string((const char*) sqlite3_column_text (*pp, 3) ?: ""); /* may be NULL */
1517 :
1518 37 : if (verbose > 1)
1519 0 : obatched(clog) << "found mtime=" << b_mtime << " stype=" << b_stype
1520 0 : << " source0=" << b_source0 << " source1=" << b_source1 << endl;
1521 :
1522 : // Try accessing the located match.
1523 : // XXX: in case of multiple matches, attempt them in parallel?
1524 37 : auto r = handle_buildid_match (b_mtime, b_stype, b_source0, b_source1, result_fd);
1525 37 : if (r)
1526 37 : return r;
1527 0 : }
1528 :
1529 : // We couldn't find it in the database. Last ditch effort
1530 : // is to defer to other debuginfo servers.
1531 :
1532 7 : int fd = -1;
1533 7 : debuginfod_client *client = debuginfod_begin ();
1534 7 : if (client != NULL)
1535 : {
1536 7 : debuginfod_set_progressfn (client, & debuginfod_find_progress);
1537 :
1538 7 : if (conn)
1539 : {
1540 : // Transcribe incoming User-Agent:
1541 14 : string ua = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";
1542 21 : string ua_complete = string("User-Agent: ") + ua;
1543 7 : debuginfod_add_http_header (client, ua_complete.c_str());
1544 :
1545 : // Compute larger XFF:, for avoiding info loss during
1546 : // federation, and for future cyclicity detection.
1547 14 : string xff = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";
1548 7 : if (xff != "")
1549 3 : xff += string(", "); // comma separated list
1550 :
1551 : // Compute the client's numeric IP address only - so can't merge with conninfo()
1552 7 : const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
1553 : MHD_CONNECTION_INFO_CLIENT_ADDRESS);
1554 7 : struct sockaddr *so = u ? u->client_addr : 0;
1555 7 : char hostname[256] = ""; // RFC1035
1556 7 : if (so && so->sa_family == AF_INET)
1557 7 : (void) getnameinfo (so, sizeof (struct sockaddr_in), hostname, sizeof (hostname), NULL, 0,
1558 : NI_NUMERICHOST);
1559 0 : else if (so && so->sa_family == AF_INET6)
1560 0 : (void) getnameinfo (so, sizeof (struct sockaddr_in6), hostname, sizeof (hostname), NULL, 0,
1561 : NI_NUMERICHOST);
1562 :
1563 21 : string xff_complete = string("X-Forwarded-For: ")+xff+string(hostname);
1564 7 : debuginfod_add_http_header (client, xff_complete.c_str());
1565 : }
1566 :
1567 7 : if (artifacttype == "debuginfo")
1568 6 : fd = debuginfod_find_debuginfo (client,
1569 6 : (const unsigned char*) buildid.c_str(),
1570 : 0, NULL);
1571 1 : else if (artifacttype == "executable")
1572 1 : fd = debuginfod_find_executable (client,
1573 1 : (const unsigned char*) buildid.c_str(),
1574 : 0, NULL);
1575 0 : else if (artifacttype == "source")
1576 0 : fd = debuginfod_find_source (client,
1577 0 : (const unsigned char*) buildid.c_str(),
1578 : 0, suffix.c_str(), NULL);
1579 : }
1580 : else
1581 0 : fd = -errno; /* Set by debuginfod_begin. */
1582 7 : debuginfod_end (client);
1583 :
1584 7 : if (fd >= 0)
1585 : {
1586 1 : inc_metric ("http_responses_total","result","upstream");
1587 : struct stat s;
1588 1 : int rc = fstat (fd, &s);
1589 1 : if (rc == 0)
1590 : {
1591 1 : auto r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd);
1592 1 : if (r)
1593 : {
1594 1 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
1595 1 : add_mhd_last_modified (r, s.st_mtime);
1596 1 : if (verbose > 1)
1597 0 : obatched(clog) << "serving file from upstream debuginfod/cache" << endl;
1598 1 : if (result_fd)
1599 1 : *result_fd = fd;
1600 1 : return r; // NB: don't close fd; libmicrohttpd will
1601 : }
1602 : }
1603 0 : close (fd);
1604 : }
1605 : else
1606 6 : switch(fd)
1607 : {
1608 4 : case -ENOSYS:
1609 4 : break;
1610 2 : case -ENOENT:
1611 2 : break;
1612 0 : default: // some more tricky error
1613 0 : throw libc_exception(-fd, "upstream debuginfod query failed");
1614 : }
1615 :
1616 6 : throw reportable_exception(MHD_HTTP_NOT_FOUND, "not found");
1617 : }
1618 :
1619 :
1620 : ////////////////////////////////////////////////////////////////////////
1621 :
1622 : static map<string,int64_t> metrics; // arbitrary data for /metrics query
1623 : // NB: store int64_t since all our metrics are integers; prometheus accepts double
1624 : static mutex metrics_lock;
1625 :
1626 : // utility function for assembling prometheus-compatible
1627 : // name="escaped-value" strings
1628 : // https://prometheus.io/docs/instrumenting/exposition_formats/
1629 : static string
1630 1456 : metric_label(const string& name, const string& value)
1631 : {
1632 1456 : string x = name + "=\"";
1633 8950 : for (auto&& c : value)
1634 7494 : switch(c)
1635 : {
1636 0 : case '\\': x += "\\\\"; break;
1637 0 : case '\"': x += "\\\""; break;
1638 0 : case '\n': x += "\\n"; break;
1639 7494 : default: x += c; break;
1640 : }
1641 1456 : x += "\"";
1642 1458 : return x;
1643 : }
1644 :
1645 :
1646 : // add prometheus-format metric name + label tuple (if any) + value
1647 :
1648 : static void
1649 6 : set_metric(const string& metric, int64_t value)
1650 : {
1651 6 : unique_lock<mutex> lock(metrics_lock);
1652 6 : metrics[metric] = value;
1653 6 : }
1654 : #if 0 /* unused */
1655 : static void
1656 : inc_metric(const string& metric)
1657 : {
1658 : unique_lock<mutex> lock(metrics_lock);
1659 : metrics[metric] ++;
1660 : }
1661 : #endif
1662 : static void
1663 176 : set_metric(const string& metric,
1664 : const string& lname, const string& lvalue,
1665 : int64_t value)
1666 : {
1667 528 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
1668 176 : unique_lock<mutex> lock(metrics_lock);
1669 176 : metrics[key] = value;
1670 176 : }
1671 :
1672 : static void
1673 892 : inc_metric(const string& metric,
1674 : const string& lname, const string& lvalue)
1675 : {
1676 2676 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
1677 892 : unique_lock<mutex> lock(metrics_lock);
1678 892 : metrics[key] ++;
1679 892 : }
1680 : static void
1681 388 : add_metric(const string& metric,
1682 : const string& lname, const string& lvalue,
1683 : int64_t value)
1684 : {
1685 1168 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
1686 390 : unique_lock<mutex> lock(metrics_lock);
1687 390 : metrics[key] += value;
1688 390 : }
1689 : #if 0
1690 : static void
1691 : add_metric(const string& metric,
1692 : int64_t value)
1693 : {
1694 : unique_lock<mutex> lock(metrics_lock);
1695 : metrics[metric] += value;
1696 : }
1697 : #endif
1698 :
1699 :
1700 : // and more for higher arity labels if needed
1701 :
1702 :
1703 : static struct MHD_Response*
1704 40 : handle_metrics (off_t* size)
1705 : {
1706 80 : stringstream o;
1707 : {
1708 80 : unique_lock<mutex> lock(metrics_lock);
1709 1536 : for (auto&& i : metrics)
1710 1496 : o << i.first << " " << i.second << endl;
1711 : }
1712 40 : const string& os = o.str();
1713 40 : MHD_Response* r = MHD_create_response_from_buffer (os.size(),
1714 40 : (void*) os.c_str(),
1715 : MHD_RESPMEM_MUST_COPY);
1716 40 : *size = os.size();
1717 40 : MHD_add_response_header (r, "Content-Type", "text/plain");
1718 80 : return r;
1719 : }
1720 :
1721 :
1722 : ////////////////////////////////////////////////////////////////////////
1723 :
1724 :
1725 : /* libmicrohttpd callback */
1726 : static int
1727 84 : handler_cb (void * /*cls*/,
1728 : struct MHD_Connection *connection,
1729 : const char *url,
1730 : const char *method,
1731 : const char * /*version*/,
1732 : const char * /*upload_data*/,
1733 : size_t * /*upload_data_size*/,
1734 : void ** /*con_cls*/)
1735 : {
1736 84 : struct MHD_Response *r = NULL;
1737 168 : string url_copy = url;
1738 :
1739 84 : int rc = MHD_NO; // mhd
1740 84 : int http_code = 500;
1741 84 : off_t http_size = -1;
1742 : struct timeval tv_start, tv_end;
1743 84 : gettimeofday (&tv_start, NULL);
1744 :
1745 : try
1746 : {
1747 84 : if (string(method) != "GET")
1748 0 : throw reportable_exception(400, "we support GET only");
1749 :
1750 : /* Start decoding the URL. */
1751 84 : size_t slash1 = url_copy.find('/', 1);
1752 168 : string url1 = url_copy.substr(0, slash1); // ok even if slash1 not found
1753 :
1754 84 : if (slash1 != string::npos && url1 == "/buildid")
1755 : {
1756 42 : size_t slash2 = url_copy.find('/', slash1+1);
1757 42 : if (slash2 == string::npos)
1758 0 : throw reportable_exception("/buildid/ webapi error, need buildid");
1759 :
1760 84 : string buildid = url_copy.substr(slash1+1, slash2-slash1-1);
1761 :
1762 42 : size_t slash3 = url_copy.find('/', slash2+1);
1763 90 : string artifacttype, suffix;
1764 42 : if (slash3 == string::npos)
1765 : {
1766 32 : artifacttype = url_copy.substr(slash2+1);
1767 32 : suffix = "";
1768 : }
1769 : else
1770 : {
1771 10 : artifacttype = url_copy.substr(slash2+1, slash3-slash2-1);
1772 10 : suffix = url_copy.substr(slash3); // include the slash in the suffix
1773 : }
1774 :
1775 42 : inc_metric("http_requests_total", "type", artifacttype);
1776 : // get the resulting fd so we can report its size
1777 : int fd;
1778 42 : r = handle_buildid(connection, buildid, artifacttype, suffix, &fd);
1779 36 : if (r)
1780 : {
1781 : struct stat fs;
1782 36 : if (fstat(fd, &fs) == 0)
1783 36 : http_size = fs.st_size;
1784 : // libmicrohttpd will close (fd);
1785 : }
1786 : }
1787 42 : else if (url1 == "/metrics")
1788 : {
1789 40 : inc_metric("http_requests_total", "type", "metrics");
1790 40 : r = handle_metrics(& http_size);
1791 : }
1792 : else
1793 2 : throw reportable_exception("webapi error, unrecognized /operation");
1794 :
1795 76 : if (r == 0)
1796 0 : throw reportable_exception("internal error, missing response");
1797 :
1798 76 : rc = MHD_queue_response (connection, MHD_HTTP_OK, r);
1799 76 : http_code = MHD_HTTP_OK;
1800 76 : MHD_destroy_response (r);
1801 : }
1802 8 : catch (const reportable_exception& e)
1803 : {
1804 8 : inc_metric("http_responses_total","result","error");
1805 8 : e.report(clog);
1806 8 : http_code = e.code;
1807 8 : http_size = e.message.size();
1808 8 : rc = e.mhd_send_response (connection);
1809 : }
1810 :
1811 84 : gettimeofday (&tv_end, NULL);
1812 84 : double deltas = (tv_end.tv_sec - tv_start.tv_sec) + (tv_end.tv_usec - tv_start.tv_usec)*0.000001;
1813 168 : obatched(clog) << conninfo(connection)
1814 : << ' ' << method << ' ' << url
1815 84 : << ' ' << http_code << ' ' << http_size
1816 84 : << ' ' << (int)(deltas*1000) << "ms"
1817 84 : << endl;
1818 :
1819 : // related prometheus metrics
1820 84 : string http_code_str = to_string(http_code);
1821 84 : if (http_size >= 0)
1822 84 : add_metric("http_responses_transfer_bytes_sum","code",http_code_str,
1823 : http_size);
1824 84 : inc_metric("http_responses_transfer_bytes_count","code",http_code_str);
1825 :
1826 84 : add_metric("http_responses_duration_milliseconds_sum","code",http_code_str,
1827 84 : deltas*1000); // prometheus prefers _seconds and floating point
1828 84 : inc_metric("http_responses_duration_milliseconds_count","code",http_code_str);
1829 :
1830 168 : return rc;
1831 : }
1832 :
1833 :
1834 : ////////////////////////////////////////////////////////////////////////
1835 : // borrowed originally from src/nm.c get_local_names()
1836 :
1837 : static void
1838 11 : dwarf_extract_source_paths (Elf *elf, set<string>& debug_sourcefiles)
1839 : noexcept // no exceptions - so we can simplify the altdbg resource release at end
1840 : {
1841 11 : Dwarf* dbg = dwarf_begin_elf (elf, DWARF_C_READ, NULL);
1842 11 : if (dbg == NULL)
1843 0 : return;
1844 :
1845 11 : Dwarf* altdbg = NULL;
1846 11 : int altdbg_fd = -1;
1847 :
1848 : // DWZ handling: if we have an unsatisfied debug-alt-link, add an
1849 : // empty string into the outgoing sourcefiles set, so the caller
1850 : // should know that our data is incomplete.
1851 : const char *alt_name_p;
1852 : const void *alt_build_id; // elfutils-owned memory
1853 11 : ssize_t sz = dwelf_dwarf_gnu_debugaltlink (dbg, &alt_name_p, &alt_build_id);
1854 11 : if (sz > 0) // got one!
1855 : {
1856 4 : string buildid;
1857 2 : unsigned char* build_id_bytes = (unsigned char*) alt_build_id;
1858 42 : for (ssize_t idx=0; idx<sz; idx++)
1859 : {
1860 40 : buildid += "0123456789abcdef"[build_id_bytes[idx] >> 4];
1861 40 : buildid += "0123456789abcdef"[build_id_bytes[idx] & 0xf];
1862 : }
1863 :
1864 2 : if (verbose > 3)
1865 0 : obatched(clog) << "Need altdebug buildid=" << buildid << endl;
1866 :
1867 : // but is it unsatisfied the normal elfutils ways?
1868 2 : Dwarf* alt = dwarf_getalt (dbg);
1869 2 : if (alt == NULL)
1870 : {
1871 : // Yup, unsatisfied the normal way. Maybe we can satisfy it
1872 : // from our own debuginfod database.
1873 : int alt_fd;
1874 2 : struct MHD_Response *r = 0;
1875 : try
1876 : {
1877 2 : r = handle_buildid (0, buildid, "debuginfo", "", &alt_fd);
1878 : }
1879 0 : catch (const reportable_exception& e)
1880 : {
1881 : // swallow exceptions
1882 : }
1883 :
1884 : // NB: this is not actually recursive! This invokes the web-query
1885 : // path, which cannot get back into the scan code paths.
1886 2 : if (r)
1887 : {
1888 : // Found it!
1889 2 : altdbg_fd = dup(alt_fd); // ok if this fails, downstream failures ok
1890 2 : alt = altdbg = dwarf_begin (altdbg_fd, DWARF_C_READ);
1891 : // NB: must close this dwarf and this fd at the bottom of the function!
1892 2 : MHD_destroy_response (r); // will close alt_fd
1893 2 : if (alt)
1894 2 : dwarf_setalt (dbg, alt);
1895 : }
1896 : }
1897 : else
1898 : {
1899 : // NB: dwarf_setalt(alt) inappropriate - already done!
1900 : // NB: altdbg will stay 0 so nothing tries to redundantly dealloc.
1901 : }
1902 :
1903 2 : if (alt)
1904 : {
1905 2 : if (verbose > 3)
1906 0 : obatched(clog) << "Resolved altdebug buildid=" << buildid << endl;
1907 : }
1908 : else // (alt == NULL) - signal possible presence of poor debuginfo
1909 : {
1910 0 : debug_sourcefiles.insert("");
1911 0 : if (verbose > 3)
1912 0 : obatched(clog) << "Unresolved altdebug buildid=" << buildid << endl;
1913 : }
1914 : }
1915 :
1916 11 : Dwarf_Off offset = 0;
1917 : Dwarf_Off old_offset;
1918 : size_t hsize;
1919 :
1920 333 : while (dwarf_nextcu (dbg, old_offset = offset, &offset, &hsize, NULL, NULL, NULL) == 0)
1921 : {
1922 : Dwarf_Die cudie_mem;
1923 322 : Dwarf_Die *cudie = dwarf_offdie (dbg, old_offset + hsize, &cudie_mem);
1924 :
1925 322 : if (cudie == NULL)
1926 1 : continue;
1927 322 : if (dwarf_tag (cudie) != DW_TAG_compile_unit)
1928 1 : continue;
1929 :
1930 321 : const char *cuname = dwarf_diename(cudie) ?: "unknown";
1931 :
1932 : Dwarf_Files *files;
1933 : size_t nfiles;
1934 321 : if (dwarf_getsrcfiles (cudie, &files, &nfiles) != 0)
1935 0 : continue;
1936 :
1937 : // extract DW_AT_comp_dir to resolve relative file names
1938 321 : const char *comp_dir = "";
1939 : const char *const *dirs;
1940 : size_t ndirs;
1941 642 : if (dwarf_getsrcdirs (files, &dirs, &ndirs) == 0 &&
1942 321 : dirs[0] != NULL)
1943 321 : comp_dir = dirs[0];
1944 321 : if (comp_dir == NULL)
1945 0 : comp_dir = "";
1946 :
1947 321 : if (verbose > 3)
1948 0 : obatched(clog) << "searching for sources for cu=" << cuname << " comp_dir=" << comp_dir
1949 0 : << " #files=" << nfiles << " #dirs=" << ndirs << endl;
1950 :
1951 321 : if (comp_dir[0] == '\0' && cuname[0] != '/')
1952 : {
1953 : // This is a common symptom for dwz-compressed debug files,
1954 : // where the altdebug file cannot be resolved.
1955 0 : if (verbose > 3)
1956 0 : obatched(clog) << "skipping cu=" << cuname << " due to empty comp_dir" << endl;
1957 0 : continue;
1958 : }
1959 :
1960 6282 : for (size_t f = 1; f < nfiles; f++)
1961 : {
1962 5961 : const char *hat = dwarf_filesrc (files, f, NULL, NULL);
1963 5961 : if (hat == NULL)
1964 0 : continue;
1965 :
1966 5961 : if (string(hat) == "<built-in>") // gcc intrinsics, don't bother record
1967 0 : continue;
1968 :
1969 5961 : string waldo;
1970 5961 : if (hat[0] == '/') // absolute
1971 4095 : waldo = (string (hat));
1972 1866 : else if (comp_dir[0] != '\0') // comp_dir relative
1973 1866 : waldo = (string (comp_dir) + string("/") + string (hat));
1974 : else
1975 : {
1976 0 : if (verbose > 3)
1977 0 : obatched(clog) << "skipping hat=" << hat << " due to empty comp_dir" << endl;
1978 0 : continue;
1979 : }
1980 :
1981 : // NB: this is the 'waldo' that a dbginfo client will have
1982 : // to supply for us to give them the file The comp_dir
1983 : // prefixing is a definite complication. Otherwise we'd
1984 : // have to return a setof comp_dirs (one per CU!) with
1985 : // corresponding filesrc[] names, instead of one absolute
1986 : // resoved set. Maybe we'll have to do that anyway. XXX
1987 :
1988 5961 : if (verbose > 4)
1989 0 : obatched(clog) << waldo
1990 0 : << (debug_sourcefiles.find(waldo)==debug_sourcefiles.end() ? " new" : " dup") << endl;
1991 :
1992 5961 : debug_sourcefiles.insert (waldo);
1993 : }
1994 : }
1995 :
1996 11 : dwarf_end(dbg);
1997 11 : if (altdbg)
1998 2 : dwarf_end(altdbg);
1999 11 : if (altdbg_fd >= 0)
2000 2 : close(altdbg_fd);
2001 : }
2002 :
2003 :
2004 :
2005 : static void
2006 57 : elf_classify (int fd, bool &executable_p, bool &debuginfo_p, string &buildid, set<string>& debug_sourcefiles)
2007 : {
2008 57 : Elf *elf = elf_begin (fd, ELF_C_READ_MMAP_PRIVATE, NULL);
2009 57 : if (elf == NULL)
2010 0 : return;
2011 :
2012 : try // catch our types of errors and clean up the Elf* object
2013 : {
2014 57 : if (elf_kind (elf) != ELF_K_ELF)
2015 : {
2016 37 : elf_end (elf);
2017 37 : return;
2018 : }
2019 :
2020 : GElf_Ehdr ehdr_storage;
2021 20 : GElf_Ehdr *ehdr = gelf_getehdr (elf, &ehdr_storage);
2022 20 : if (ehdr == NULL)
2023 : {
2024 0 : elf_end (elf);
2025 0 : return;
2026 : }
2027 20 : auto elf_type = ehdr->e_type;
2028 :
2029 : const void *build_id; // elfutils-owned memory
2030 20 : ssize_t sz = dwelf_elf_gnu_build_id (elf, & build_id);
2031 20 : if (sz <= 0)
2032 : {
2033 : // It's not a diagnostic-worthy error for an elf file to lack build-id.
2034 : // It might just be very old.
2035 0 : elf_end (elf);
2036 0 : return;
2037 : }
2038 :
2039 : // build_id is a raw byte array; convert to hexadecimal *lowercase*
2040 20 : unsigned char* build_id_bytes = (unsigned char*) build_id;
2041 420 : for (ssize_t idx=0; idx<sz; idx++)
2042 : {
2043 400 : buildid += "0123456789abcdef"[build_id_bytes[idx] >> 4];
2044 400 : buildid += "0123456789abcdef"[build_id_bytes[idx] & 0xf];
2045 : }
2046 :
2047 : // now decide whether it's an executable - namely, any allocatable section has
2048 : // PROGBITS;
2049 20 : if (elf_type == ET_EXEC || elf_type == ET_DYN)
2050 : {
2051 : size_t shnum;
2052 18 : int rc = elf_getshdrnum (elf, &shnum);
2053 18 : if (rc < 0)
2054 0 : throw elfutils_exception(rc, "getshdrnum");
2055 :
2056 18 : executable_p = false;
2057 317 : for (size_t sc = 0; sc < shnum; sc++)
2058 : {
2059 309 : Elf_Scn *scn = elf_getscn (elf, sc);
2060 309 : if (scn == NULL)
2061 0 : continue;
2062 :
2063 : GElf_Shdr shdr_mem;
2064 309 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_mem);
2065 309 : if (shdr == NULL)
2066 0 : continue;
2067 :
2068 : // allocated (loadable / vm-addr-assigned) section with available content?
2069 309 : if ((shdr->sh_type == SHT_PROGBITS) && (shdr->sh_flags & SHF_ALLOC))
2070 : {
2071 10 : if (verbose > 4)
2072 0 : obatched(clog) << "executable due to SHF_ALLOC SHT_PROGBITS sc=" << sc << endl;
2073 10 : executable_p = true;
2074 10 : break; // no need to keep looking for others
2075 : }
2076 : } // iterate over sections
2077 : } // executable_p classification
2078 :
2079 : // now decide whether it's a debuginfo - namely, if it has any .debug* or .zdebug* sections
2080 : // logic mostly stolen from fweimer@redhat.com's elfclassify drafts
2081 : size_t shstrndx;
2082 20 : int rc = elf_getshdrstrndx (elf, &shstrndx);
2083 20 : if (rc < 0)
2084 0 : throw elfutils_exception(rc, "getshdrstrndx");
2085 :
2086 20 : Elf_Scn *scn = NULL;
2087 : while (true)
2088 : {
2089 507 : scn = elf_nextscn (elf, scn);
2090 506 : if (scn == NULL)
2091 20 : break;
2092 : GElf_Shdr shdr_storage;
2093 497 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_storage);
2094 498 : if (shdr == NULL)
2095 0 : break;
2096 498 : const char *section_name = elf_strptr (elf, shstrndx, shdr->sh_name);
2097 498 : if (section_name == NULL)
2098 0 : break;
2099 498 : if (strncmp(section_name, ".debug_line", 11) == 0 ||
2100 487 : strncmp(section_name, ".zdebug_line", 12) == 0)
2101 : {
2102 11 : debuginfo_p = true;
2103 11 : dwarf_extract_source_paths (elf, debug_sourcefiles);
2104 11 : break; // expecting only one .*debug_line, so no need to look for others
2105 : }
2106 487 : else if (strncmp(section_name, ".debug_", 7) == 0 ||
2107 455 : strncmp(section_name, ".zdebug_", 8) == 0)
2108 : {
2109 35 : debuginfo_p = true;
2110 : // NB: don't break; need to parse .debug_line for sources
2111 : }
2112 487 : }
2113 : }
2114 0 : catch (const reportable_exception& e)
2115 : {
2116 0 : e.report(clog);
2117 : }
2118 20 : elf_end (elf);
2119 : }
2120 :
2121 :
2122 : static void
2123 54 : scan_source_file (const string& rps, const stat_t& st,
2124 : sqlite_ps& ps_upsert_buildids,
2125 : sqlite_ps& ps_upsert_files,
2126 : sqlite_ps& ps_upsert_de,
2127 : sqlite_ps& ps_upsert_s,
2128 : sqlite_ps& ps_query,
2129 : sqlite_ps& ps_scan_done,
2130 : unsigned& fts_cached,
2131 : unsigned& fts_executable,
2132 : unsigned& fts_debuginfo,
2133 : unsigned& fts_sourcefiles)
2134 : {
2135 : /* See if we know of it already. */
2136 : int rc = ps_query
2137 54 : .reset()
2138 54 : .bind(1, rps)
2139 54 : .bind(2, st.st_mtime)
2140 54 : .step();
2141 54 : ps_query.reset();
2142 54 : if (rc == SQLITE_ROW) // i.e., a result, as opposed to DONE (no results)
2143 : // no need to recheck a file/version we already know
2144 : // specifically, no need to elf-begin a file we already determined is non-elf
2145 : // (so is stored with buildid=NULL)
2146 : {
2147 29 : fts_cached++;
2148 29 : return;
2149 : }
2150 :
2151 25 : bool executable_p = false, debuginfo_p = false; // E and/or D
2152 50 : string buildid;
2153 50 : set<string> sourcefiles;
2154 :
2155 25 : int fd = open (rps.c_str(), O_RDONLY);
2156 : try
2157 : {
2158 25 : if (fd >= 0)
2159 25 : elf_classify (fd, executable_p, debuginfo_p, buildid, sourcefiles);
2160 : else
2161 0 : throw libc_exception(errno, string("open ") + rps);
2162 25 : inc_metric ("scanned_total","source","file");
2163 : }
2164 : // NB: we catch exceptions here too, so that we can
2165 : // cache the corrupt-elf case (!executable_p &&
2166 : // !debuginfo_p) just below, just as if we had an
2167 : // EPERM error from open(2).
2168 0 : catch (const reportable_exception& e)
2169 : {
2170 0 : e.report(clog);
2171 : }
2172 :
2173 25 : if (fd >= 0)
2174 25 : close (fd);
2175 :
2176 : // register this file name in the interning table
2177 : ps_upsert_files
2178 25 : .reset()
2179 25 : .bind(1, rps)
2180 25 : .step_ok_done();
2181 :
2182 25 : if (buildid == "")
2183 : {
2184 : // no point storing an elf file without buildid
2185 21 : executable_p = false;
2186 21 : debuginfo_p = false;
2187 : }
2188 : else
2189 : {
2190 : // register this build-id in the interning table
2191 : ps_upsert_buildids
2192 4 : .reset()
2193 4 : .bind(1, buildid)
2194 4 : .step_ok_done();
2195 : }
2196 :
2197 25 : if (executable_p)
2198 3 : fts_executable ++;
2199 25 : if (debuginfo_p)
2200 3 : fts_debuginfo ++;
2201 25 : if (executable_p || debuginfo_p)
2202 : {
2203 : ps_upsert_de
2204 4 : .reset()
2205 4 : .bind(1, buildid)
2206 4 : .bind(2, debuginfo_p ? 1 : 0)
2207 4 : .bind(3, executable_p ? 1 : 0)
2208 4 : .bind(4, rps)
2209 4 : .bind(5, st.st_mtime)
2210 4 : .step_ok_done();
2211 : }
2212 25 : if (executable_p)
2213 3 : inc_metric("found_executable_total","source","files");
2214 25 : if (debuginfo_p)
2215 3 : inc_metric("found_debuginfo_total","source","files");
2216 :
2217 25 : if (sourcefiles.size() && buildid != "")
2218 : {
2219 3 : fts_sourcefiles += sourcefiles.size();
2220 :
2221 493 : for (auto&& dwarfsrc : sourcefiles)
2222 : {
2223 490 : char *srp = realpath(dwarfsrc.c_str(), NULL);
2224 490 : if (srp == NULL) // also if DWZ unresolved dwarfsrc=""
2225 6 : continue; // unresolvable files are not a serious problem
2226 : // throw libc_exception(errno, "fts/file realpath " + srcpath);
2227 484 : string srps = string(srp);
2228 484 : free (srp);
2229 :
2230 : struct stat sfs;
2231 484 : rc = stat(srps.c_str(), &sfs);
2232 484 : if (rc != 0)
2233 0 : continue;
2234 :
2235 484 : if (verbose > 2)
2236 0 : obatched(clog) << "recorded buildid=" << buildid << " file=" << srps
2237 0 : << " mtime=" << sfs.st_mtime
2238 0 : << " as source " << dwarfsrc << endl;
2239 :
2240 : ps_upsert_files
2241 484 : .reset()
2242 484 : .bind(1, srps)
2243 484 : .step_ok_done();
2244 :
2245 : // register the dwarfsrc name in the interning table too
2246 : ps_upsert_files
2247 484 : .reset()
2248 484 : .bind(1, dwarfsrc)
2249 484 : .step_ok_done();
2250 :
2251 : ps_upsert_s
2252 484 : .reset()
2253 484 : .bind(1, buildid)
2254 484 : .bind(2, dwarfsrc)
2255 484 : .bind(3, srps)
2256 484 : .bind(4, sfs.st_mtime)
2257 484 : .step_ok_done();
2258 :
2259 : // PR25548: also store canonicalized source path
2260 484 : string dwarfsrc_canon = canon_pathname (dwarfsrc);
2261 484 : if (dwarfsrc_canon != dwarfsrc)
2262 : {
2263 84 : if (verbose > 3)
2264 0 : obatched(clog) << "canonicalized src=" << dwarfsrc << " alias=" << dwarfsrc_canon << endl;
2265 :
2266 : ps_upsert_files
2267 84 : .reset()
2268 84 : .bind(1, dwarfsrc_canon)
2269 84 : .step_ok_done();
2270 :
2271 : ps_upsert_s
2272 84 : .reset()
2273 84 : .bind(1, buildid)
2274 84 : .bind(2, dwarfsrc_canon)
2275 84 : .bind(3, srps)
2276 84 : .bind(4, sfs.st_mtime)
2277 84 : .step_ok_done();
2278 : }
2279 :
2280 484 : inc_metric("found_sourcerefs_total","source","files");
2281 : }
2282 : }
2283 :
2284 : ps_scan_done
2285 25 : .reset()
2286 25 : .bind(1, rps)
2287 25 : .bind(2, st.st_mtime)
2288 25 : .bind(3, st.st_size)
2289 25 : .step_ok_done();
2290 :
2291 25 : if (verbose > 2)
2292 0 : obatched(clog) << "recorded buildid=" << buildid << " file=" << rps
2293 0 : << " mtime=" << st.st_mtime << " atype="
2294 : << (executable_p ? "E" : "")
2295 0 : << (debuginfo_p ? "D" : "") << endl;
2296 : }
2297 :
2298 :
2299 :
2300 :
2301 :
2302 : // Analyze given archive file of given age; record buildids / exec/debuginfo-ness of its
2303 : // constituent files with given upsert statements.
2304 : static void
2305 16 : archive_classify (const string& rps, string& archive_extension,
2306 : sqlite_ps& ps_upsert_buildids, sqlite_ps& ps_upsert_files,
2307 : sqlite_ps& ps_upsert_de, sqlite_ps& ps_upsert_sref, sqlite_ps& ps_upsert_sdef,
2308 : time_t mtime,
2309 : unsigned& fts_executable, unsigned& fts_debuginfo, unsigned& fts_sref, unsigned& fts_sdef,
2310 : bool& fts_sref_complete_p)
2311 : {
2312 32 : string archive_decoder = "/dev/null";
2313 64 : for (auto&& arch : scan_archives)
2314 48 : if (string_endswith(rps, arch.first))
2315 : {
2316 16 : archive_extension = arch.first;
2317 16 : archive_decoder = arch.second;
2318 : }
2319 :
2320 : FILE* fp;
2321 : defer_dtor<FILE*,int>::dtor_fn dfn;
2322 16 : if (archive_decoder != "cat")
2323 : {
2324 3 : string popen_cmd = archive_decoder + " " + shell_escape(rps);
2325 1 : fp = popen (popen_cmd.c_str(), "r"); // "e" O_CLOEXEC?
2326 1 : dfn = pclose;
2327 1 : if (fp == NULL)
2328 0 : throw libc_exception (errno, string("popen ") + popen_cmd);
2329 : }
2330 : else
2331 : {
2332 15 : fp = fopen (rps.c_str(), "r");
2333 15 : dfn = fclose;
2334 15 : if (fp == NULL)
2335 0 : throw libc_exception (errno, string("fopen ") + rps);
2336 : }
2337 32 : defer_dtor<FILE*,int> fp_closer (fp, dfn);
2338 :
2339 : struct archive *a;
2340 16 : a = archive_read_new();
2341 15 : if (a == NULL)
2342 0 : throw archive_exception("cannot create archive reader");
2343 15 : defer_dtor<struct archive*,int> archive_closer (a, archive_read_free);
2344 :
2345 14 : int rc = archive_read_support_format_all(a);
2346 16 : if (rc != ARCHIVE_OK)
2347 0 : throw archive_exception(a, "cannot select all formats");
2348 16 : rc = archive_read_support_filter_all(a);
2349 14 : if (rc != ARCHIVE_OK)
2350 0 : throw archive_exception(a, "cannot select all filters");
2351 :
2352 14 : rc = archive_read_open_FILE (a, fp);
2353 16 : if (rc != ARCHIVE_OK)
2354 0 : throw archive_exception(a, "cannot open archive from pipe");
2355 :
2356 16 : if (verbose > 3)
2357 0 : obatched(clog) << "libarchive scanning " << rps << endl;
2358 :
2359 : while(1) // parse cpio archive entries
2360 : {
2361 : try
2362 : {
2363 : struct archive_entry *e;
2364 112 : rc = archive_read_next_header (a, &e);
2365 112 : if (rc != ARCHIVE_OK)
2366 16 : break;
2367 :
2368 96 : if (! S_ISREG(archive_entry_mode (e))) // skip non-files completely
2369 64 : continue;
2370 :
2371 63 : string fn = canonicalized_archive_entry_pathname (e);
2372 :
2373 32 : if (verbose > 3)
2374 0 : obatched(clog) << "libarchive checking " << fn << endl;
2375 :
2376 : // extract this file to a temporary file
2377 32 : char* tmppath = NULL;
2378 32 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
2379 32 : if (rc < 0)
2380 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
2381 64 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
2382 31 : int fd = mkstemp (tmppath);
2383 32 : if (fd < 0)
2384 0 : throw libc_exception (errno, "cannot create temporary file");
2385 32 : unlink (tmppath); // unlink now so OS will release the file as soon as we close the fd
2386 64 : defer_dtor<int,int> minifd_closer (fd, close);
2387 :
2388 32 : rc = archive_read_data_into_fd (a, fd);
2389 32 : if (rc != ARCHIVE_OK)
2390 0 : throw archive_exception(a, "cannot extract file");
2391 :
2392 : // finally ... time to run elf_classify on this bad boy and update the database
2393 32 : bool executable_p = false, debuginfo_p = false;
2394 64 : string buildid;
2395 64 : set<string> sourcefiles;
2396 32 : elf_classify (fd, executable_p, debuginfo_p, buildid, sourcefiles);
2397 : // NB: might throw
2398 :
2399 32 : if (buildid != "") // intern buildid
2400 : {
2401 : ps_upsert_buildids
2402 16 : .reset()
2403 16 : .bind(1, buildid)
2404 16 : .step_ok_done();
2405 : }
2406 :
2407 : ps_upsert_files // register this rpm constituent file name in interning table
2408 32 : .reset()
2409 32 : .bind(1, fn)
2410 32 : .step_ok_done();
2411 :
2412 32 : if (sourcefiles.size() > 0) // sref records needed
2413 : {
2414 : // NB: we intern each source file once. Once raw, as it
2415 : // appears in the DWARF file list coming back from
2416 : // elf_classify() - because it'll end up in the
2417 : // _norm.artifactsrc column. We don't also put another
2418 : // version with a '.' at the front, even though that's
2419 : // how rpm/cpio packs names, because we hide that from
2420 : // the database for storage efficiency.
2421 :
2422 28 : for (auto&& s : sourcefiles)
2423 : {
2424 21 : if (s == "")
2425 : {
2426 0 : fts_sref_complete_p = false;
2427 0 : continue;
2428 : }
2429 :
2430 : ps_upsert_files
2431 21 : .reset()
2432 21 : .bind(1, s)
2433 21 : .step_ok_done();
2434 :
2435 : ps_upsert_sref
2436 21 : .reset()
2437 21 : .bind(1, buildid)
2438 21 : .bind(2, s)
2439 21 : .step_ok_done();
2440 :
2441 : // PR25548: also store canonicalized source path
2442 21 : const string& dwarfsrc = s;
2443 21 : string dwarfsrc_canon = canon_pathname (dwarfsrc);
2444 21 : if (dwarfsrc_canon != dwarfsrc)
2445 : {
2446 0 : if (verbose > 3)
2447 0 : obatched(clog) << "canonicalized src=" << dwarfsrc << " alias=" << dwarfsrc_canon << endl;
2448 :
2449 : ps_upsert_files
2450 0 : .reset()
2451 0 : .bind(1, dwarfsrc_canon)
2452 0 : .step_ok_done();
2453 :
2454 : ps_upsert_sref
2455 0 : .reset()
2456 0 : .bind(1, buildid)
2457 0 : .bind(2, dwarfsrc_canon)
2458 0 : .step_ok_done();
2459 : }
2460 :
2461 21 : fts_sref ++;
2462 : }
2463 : }
2464 :
2465 32 : if (executable_p)
2466 7 : fts_executable ++;
2467 32 : if (debuginfo_p)
2468 9 : fts_debuginfo ++;
2469 :
2470 32 : if (executable_p || debuginfo_p)
2471 : {
2472 : ps_upsert_de
2473 16 : .reset()
2474 16 : .bind(1, buildid)
2475 16 : .bind(2, debuginfo_p ? 1 : 0)
2476 16 : .bind(3, executable_p ? 1 : 0)
2477 16 : .bind(4, rps)
2478 16 : .bind(5, mtime)
2479 16 : .bind(6, fn)
2480 16 : .step_ok_done();
2481 : }
2482 : else // potential source - sdef record
2483 : {
2484 16 : fts_sdef ++;
2485 : ps_upsert_sdef
2486 16 : .reset()
2487 16 : .bind(1, rps)
2488 16 : .bind(2, mtime)
2489 16 : .bind(3, fn)
2490 16 : .step_ok_done();
2491 : }
2492 :
2493 32 : if ((verbose > 2) && (executable_p || debuginfo_p))
2494 0 : obatched(clog) << "recorded buildid=" << buildid << " rpm=" << rps << " file=" << fn
2495 0 : << " mtime=" << mtime << " atype="
2496 : << (executable_p ? "E" : "")
2497 : << (debuginfo_p ? "D" : "")
2498 0 : << " sourcefiles=" << sourcefiles.size() << endl;
2499 :
2500 : }
2501 0 : catch (const reportable_exception& e)
2502 : {
2503 0 : e.report(clog);
2504 : }
2505 96 : }
2506 16 : }
2507 :
2508 :
2509 :
2510 : // scan for archive files such as .rpm
2511 : static void
2512 32 : scan_archive_file (const string& rps, const stat_t& st,
2513 : sqlite_ps& ps_upsert_buildids,
2514 : sqlite_ps& ps_upsert_files,
2515 : sqlite_ps& ps_upsert_de,
2516 : sqlite_ps& ps_upsert_sref,
2517 : sqlite_ps& ps_upsert_sdef,
2518 : sqlite_ps& ps_query,
2519 : sqlite_ps& ps_scan_done,
2520 : unsigned& fts_cached,
2521 : unsigned& fts_executable,
2522 : unsigned& fts_debuginfo,
2523 : unsigned& fts_sref,
2524 : unsigned& fts_sdef)
2525 : {
2526 : /* See if we know of it already. */
2527 : int rc = ps_query
2528 32 : .reset()
2529 32 : .bind(1, rps)
2530 32 : .bind(2, st.st_mtime)
2531 32 : .step();
2532 32 : ps_query.reset();
2533 32 : if (rc == SQLITE_ROW) // i.e., a result, as opposed to DONE (no results)
2534 : // no need to recheck a file/version we already know
2535 : // specifically, no need to parse this archive again, since we already have
2536 : // it as a D or E or S record,
2537 : // (so is stored with buildid=NULL)
2538 : {
2539 16 : fts_cached ++;
2540 16 : return;
2541 : }
2542 :
2543 : // intern the archive file name
2544 : ps_upsert_files
2545 16 : .reset()
2546 16 : .bind(1, rps)
2547 16 : .step_ok_done();
2548 :
2549 : // extract the archive contents
2550 16 : unsigned my_fts_executable = 0, my_fts_debuginfo = 0, my_fts_sref = 0, my_fts_sdef = 0;
2551 16 : bool my_fts_sref_complete_p = true;
2552 : try
2553 : {
2554 16 : string archive_extension;
2555 16 : archive_classify (rps, archive_extension,
2556 : ps_upsert_buildids, ps_upsert_files,
2557 : ps_upsert_de, ps_upsert_sref, ps_upsert_sdef, // dalt
2558 16 : st.st_mtime,
2559 : my_fts_executable, my_fts_debuginfo, my_fts_sref, my_fts_sdef,
2560 : my_fts_sref_complete_p);
2561 16 : inc_metric ("scanned_total","source",archive_extension + " archive");
2562 16 : add_metric("found_debuginfo_total","source",archive_extension + " archive",
2563 : my_fts_debuginfo);
2564 16 : add_metric("found_executable_total","source",archive_extension + " archive",
2565 : my_fts_executable);
2566 16 : add_metric("found_sourcerefs_total","source",archive_extension + " archive",
2567 : my_fts_sref);
2568 : }
2569 0 : catch (const reportable_exception& e)
2570 : {
2571 0 : e.report(clog);
2572 : }
2573 :
2574 16 : if (verbose > 2)
2575 0 : obatched(clog) << "scanned archive=" << rps
2576 0 : << " mtime=" << st.st_mtime
2577 0 : << " executables=" << my_fts_executable
2578 0 : << " debuginfos=" << my_fts_debuginfo
2579 0 : << " srefs=" << my_fts_sref
2580 0 : << " sdefs=" << my_fts_sdef
2581 0 : << endl;
2582 :
2583 16 : fts_executable += my_fts_executable;
2584 16 : fts_debuginfo += my_fts_debuginfo;
2585 16 : fts_sref += my_fts_sref;
2586 16 : fts_sdef += my_fts_sdef;
2587 :
2588 16 : if (my_fts_sref_complete_p) // leave incomplete?
2589 : ps_scan_done
2590 16 : .reset()
2591 16 : .bind(1, rps)
2592 16 : .bind(2, st.st_mtime)
2593 16 : .bind(3, st.st_size)
2594 16 : .step_ok_done();
2595 : }
2596 :
2597 :
2598 :
2599 : ////////////////////////////////////////////////////////////////////////
2600 :
2601 :
2602 :
2603 : // The thread that consumes file names off of the scanq. We hold
2604 : // the persistent sqlite_ps's at this level and delegate file/archive
2605 : // scanning to other functions.
2606 : static void*
2607 11 : thread_main_scanner (void* arg)
2608 : {
2609 : (void) arg;
2610 :
2611 : // all the prepared statements fit to use, the _f_ set:
2612 35 : sqlite_ps ps_f_upsert_buildids (db, "file-buildids-intern", "insert or ignore into " BUILDIDS "_buildids VALUES (NULL, ?);");
2613 36 : sqlite_ps ps_f_upsert_files (db, "file-files-intern", "insert or ignore into " BUILDIDS "_files VALUES (NULL, ?);");
2614 : sqlite_ps ps_f_upsert_de (db, "file-de-upsert",
2615 : "insert or ignore into " BUILDIDS "_f_de "
2616 : "(buildid, debuginfo_p, executable_p, file, mtime) "
2617 : "values ((select id from " BUILDIDS "_buildids where hex = ?),"
2618 : " ?,?,"
2619 36 : " (select id from " BUILDIDS "_files where name = ?), ?);");
2620 : sqlite_ps ps_f_upsert_s (db, "file-s-upsert",
2621 : "insert or ignore into " BUILDIDS "_f_s "
2622 : "(buildid, artifactsrc, file, mtime) "
2623 : "values ((select id from " BUILDIDS "_buildids where hex = ?),"
2624 : " (select id from " BUILDIDS "_files where name = ?),"
2625 : " (select id from " BUILDIDS "_files where name = ?),"
2626 36 : " ?);");
2627 : sqlite_ps ps_f_query (db, "file-negativehit-find",
2628 : "select 1 from " BUILDIDS "_file_mtime_scanned where sourcetype = 'F' "
2629 36 : "and file = (select id from " BUILDIDS "_files where name = ?) and mtime = ?;");
2630 : sqlite_ps ps_f_scan_done (db, "file-scanned",
2631 : "insert or ignore into " BUILDIDS "_file_mtime_scanned (sourcetype, file, mtime, size)"
2632 36 : "values ('F', (select id from " BUILDIDS "_files where name = ?), ?, ?);");
2633 :
2634 : // and now for the _r_ set
2635 36 : sqlite_ps ps_r_upsert_buildids (db, "rpm-buildid-intern", "insert or ignore into " BUILDIDS "_buildids VALUES (NULL, ?);");
2636 36 : sqlite_ps ps_r_upsert_files (db, "rpm-file-intern", "insert or ignore into " BUILDIDS "_files VALUES (NULL, ?);");
2637 : sqlite_ps ps_r_upsert_de (db, "rpm-de-insert",
2638 : "insert or ignore into " BUILDIDS "_r_de (buildid, debuginfo_p, executable_p, file, mtime, content) values ("
2639 : "(select id from " BUILDIDS "_buildids where hex = ?), ?, ?, "
2640 : "(select id from " BUILDIDS "_files where name = ?), ?, "
2641 36 : "(select id from " BUILDIDS "_files where name = ?));");
2642 : sqlite_ps ps_r_upsert_sref (db, "rpm-sref-insert",
2643 : "insert or ignore into " BUILDIDS "_r_sref (buildid, artifactsrc) values ("
2644 : "(select id from " BUILDIDS "_buildids where hex = ?), "
2645 36 : "(select id from " BUILDIDS "_files where name = ?));");
2646 : sqlite_ps ps_r_upsert_sdef (db, "rpm-sdef-insert",
2647 : "insert or ignore into " BUILDIDS "_r_sdef (file, mtime, content) values ("
2648 : "(select id from " BUILDIDS "_files where name = ?), ?,"
2649 36 : "(select id from " BUILDIDS "_files where name = ?));");
2650 : sqlite_ps ps_r_query (db, "rpm-negativehit-query",
2651 : "select 1 from " BUILDIDS "_file_mtime_scanned where "
2652 36 : "sourcetype = 'R' and file = (select id from " BUILDIDS "_files where name = ?) and mtime = ?;");
2653 : sqlite_ps ps_r_scan_done (db, "rpm-scanned",
2654 : "insert or ignore into " BUILDIDS "_file_mtime_scanned (sourcetype, file, mtime, size)"
2655 24 : "values ('R', (select id from " BUILDIDS "_files where name = ?), ?, ?);");
2656 :
2657 :
2658 12 : unsigned fts_cached = 0, fts_executable = 0, fts_debuginfo = 0, fts_sourcefiles = 0;
2659 12 : unsigned fts_sref = 0, fts_sdef = 0;
2660 :
2661 12 : add_metric("thread_count", "role", "scan", 1);
2662 12 : add_metric("thread_busy", "role", "scan", 1);
2663 78 : while (! interrupted)
2664 : {
2665 66 : scan_payload p;
2666 :
2667 66 : add_metric("thread_busy", "role", "scan", -1);
2668 66 : bool gotone = scanq.wait_front(p);
2669 65 : add_metric("thread_busy", "role", "scan", 1);
2670 66 : if (! gotone) continue; // or break
2671 :
2672 : try
2673 : {
2674 54 : bool scan_archive = false;
2675 215 : for (auto&& arch : scan_archives)
2676 161 : if (string_endswith(p.first, arch.first))
2677 32 : scan_archive = true;
2678 :
2679 54 : if (scan_archive)
2680 32 : scan_archive_file (p.first, p.second,
2681 : ps_r_upsert_buildids,
2682 : ps_r_upsert_files,
2683 : ps_r_upsert_de,
2684 : ps_r_upsert_sref,
2685 : ps_r_upsert_sdef,
2686 : ps_r_query,
2687 : ps_r_scan_done,
2688 : fts_cached,
2689 : fts_executable,
2690 : fts_debuginfo,
2691 : fts_sref,
2692 : fts_sdef);
2693 :
2694 54 : if (scan_files) // NB: maybe "else if" ?
2695 54 : scan_source_file (p.first, p.second,
2696 : ps_f_upsert_buildids,
2697 : ps_f_upsert_files,
2698 : ps_f_upsert_de,
2699 : ps_f_upsert_s,
2700 : ps_f_query,
2701 : ps_f_scan_done,
2702 : fts_cached, fts_executable, fts_debuginfo, fts_sourcefiles);
2703 : }
2704 0 : catch (const reportable_exception& e)
2705 : {
2706 0 : e.report(cerr);
2707 : }
2708 :
2709 54 : inc_metric("thread_work_total", "role","scan");
2710 : }
2711 :
2712 12 : add_metric("thread_busy", "role", "scan", -1);
2713 24 : return 0;
2714 : }
2715 :
2716 :
2717 :
2718 : // The thread that traverses all the source_paths and enqueues all the
2719 : // matching files into the file/archive scan queue.
2720 : static void
2721 7 : scan_source_paths()
2722 : {
2723 : // NB: fedora 31 glibc/fts(3) crashes inside fts_read() on empty
2724 : // path list.
2725 7 : if (source_paths.empty())
2726 1 : return;
2727 :
2728 : // Turn the source_paths into an fts(3)-compatible char**. Since
2729 : // source_paths[] does not change after argv processing, the
2730 : // c_str()'s are safe to keep around awile.
2731 12 : vector<const char *> sps;
2732 28 : for (auto&& sp: source_paths)
2733 22 : sps.push_back(sp.c_str());
2734 6 : sps.push_back(NULL);
2735 :
2736 6 : FTS *fts = fts_open ((char * const *)sps.data(),
2737 : (traverse_logical ? FTS_LOGICAL : FTS_PHYSICAL|FTS_XDEV)
2738 : | FTS_NOCHDIR /* multithreaded */,
2739 : NULL);
2740 6 : if (fts == NULL)
2741 0 : throw libc_exception(errno, "cannot fts_open");
2742 6 : defer_dtor<FTS*,int> fts_cleanup (fts, fts_close);
2743 :
2744 : struct timeval tv_start, tv_end;
2745 6 : gettimeofday (&tv_start, NULL);
2746 6 : unsigned fts_scanned = 0, fts_regex = 0;
2747 :
2748 : FTSENT *f;
2749 133 : while ((f = fts_read (fts)) != NULL)
2750 : {
2751 127 : if (interrupted) break;
2752 :
2753 127 : fts_scanned ++;
2754 :
2755 127 : if (verbose > 2)
2756 0 : obatched(clog) << "fts traversing " << f->fts_path << endl;
2757 :
2758 : /* Found a file. Convert it to an absolute path, so
2759 : the buildid database does not have relative path
2760 : names that are unresolvable from a subsequent run
2761 : in a different cwd. */
2762 127 : char *rp = realpath(f->fts_path, NULL);
2763 127 : if (rp == NULL)
2764 0 : continue; // ignore dangling symlink or such
2765 127 : string rps = string(rp);
2766 127 : free (rp);
2767 :
2768 127 : bool ri = !regexec (&file_include_regex, rps.c_str(), 0, 0, 0);
2769 127 : bool rx = !regexec (&file_exclude_regex, rps.c_str(), 0, 0, 0);
2770 127 : if (!ri || rx)
2771 : {
2772 0 : if (verbose > 3)
2773 0 : obatched(clog) << "fts skipped by regex " << (!ri ? "I" : "") << (rx ? "X" : "") << endl;
2774 0 : fts_regex ++;
2775 0 : continue;
2776 : }
2777 :
2778 127 : switch (f->fts_info)
2779 : {
2780 54 : case FTS_F:
2781 54 : scanq.push_back (make_pair(rps, *f->fts_statp));
2782 54 : break;
2783 :
2784 0 : case FTS_ERR:
2785 : case FTS_NS:
2786 : // report on some types of errors because they may reflect fixable misconfiguration
2787 : {
2788 0 : auto x = libc_exception(f->fts_errno, string("fts traversal ") + string(f->fts_path));
2789 0 : x.report(cerr);
2790 : }
2791 0 : break;
2792 :
2793 127 : default:
2794 : ;
2795 : /* ignore */
2796 : }
2797 : }
2798 6 : gettimeofday (&tv_end, NULL);
2799 6 : double deltas = (tv_end.tv_sec - tv_start.tv_sec) + (tv_end.tv_usec - tv_start.tv_usec)*0.000001;
2800 :
2801 12 : obatched(clog) << "fts traversed source paths in " << deltas << "s, scanned=" << fts_scanned
2802 6 : << ", regex-skipped=" << fts_regex << endl;
2803 : }
2804 :
2805 :
2806 : static void*
2807 3 : thread_main_fts_source_paths (void* arg)
2808 : {
2809 : (void) arg; // ignore; we operate on global data
2810 :
2811 3 : sig_atomic_t forced_rescan_count = 0;
2812 3 : set_metric("thread_tid", "role","traverse", tid());
2813 3 : add_metric("thread_count", "role", "traverse", 1);
2814 :
2815 3 : time_t last_rescan = 0;
2816 :
2817 13 : while (! interrupted)
2818 : {
2819 13 : sleep (1);
2820 13 : scanq.wait_idle(); // don't start a new traversal while scanners haven't finished the job
2821 13 : scanq.done_idle(); // release the hounds
2822 13 : if (interrupted) break;
2823 :
2824 10 : time_t now = time(NULL);
2825 10 : bool rescan_now = false;
2826 10 : if (last_rescan == 0) // at least one initial rescan is documented even for -t0
2827 3 : rescan_now = true;
2828 10 : if (rescan_s > 0 && (long)now > (long)(last_rescan + rescan_s))
2829 2 : rescan_now = true;
2830 10 : if (sigusr1 != forced_rescan_count)
2831 : {
2832 4 : forced_rescan_count = sigusr1;
2833 4 : rescan_now = true;
2834 : }
2835 10 : if (rescan_now)
2836 : try
2837 : {
2838 7 : set_metric("thread_busy", "role","traverse", 1);
2839 7 : scan_source_paths();
2840 7 : last_rescan = time(NULL); // NB: now was before scanning
2841 7 : inc_metric("thread_work_total", "role","traverse");
2842 7 : set_metric("thread_busy", "role","traverse", 0);
2843 : }
2844 0 : catch (const reportable_exception& e)
2845 : {
2846 0 : e.report(cerr);
2847 : }
2848 : }
2849 :
2850 3 : return 0;
2851 : }
2852 :
2853 :
2854 :
2855 : ////////////////////////////////////////////////////////////////////////
2856 :
2857 : static void
2858 4 : database_stats_report()
2859 : {
2860 : sqlite_ps ps_query (db, "database-overview",
2861 12 : "select label,quantity from " BUILDIDS "_stats");
2862 :
2863 4 : obatched(clog) << "database record counts:" << endl;
2864 : while (1)
2865 : {
2866 44 : int rc = sqlite3_step (ps_query);
2867 44 : if (rc == SQLITE_DONE) break;
2868 40 : if (rc != SQLITE_ROW)
2869 0 : throw sqlite_exception(rc, "step");
2870 :
2871 80 : obatched(clog)
2872 80 : << right << setw(20) << ((const char*) sqlite3_column_text(ps_query, 0) ?: (const char*) "NULL")
2873 : << " "
2874 80 : << (sqlite3_column_text(ps_query, 1) ?: (const unsigned char*) "NULL")
2875 40 : << endl;
2876 :
2877 40 : set_metric("groom", "statistic",
2878 40 : ((const char*) sqlite3_column_text(ps_query, 0) ?: (const char*) "NULL"),
2879 40 : (sqlite3_column_double(ps_query, 1)));
2880 40 : }
2881 4 : }
2882 :
2883 :
2884 : // Do a round of database grooming that might take many minutes to run.
2885 4 : void groom()
2886 : {
2887 4 : obatched(clog) << "grooming database" << endl;
2888 :
2889 : struct timeval tv_start, tv_end;
2890 4 : gettimeofday (&tv_start, NULL);
2891 :
2892 : // scan for files that have disappeared
2893 : sqlite_ps files (db, "check old files", "select s.mtime, s.file, f.name from "
2894 : BUILDIDS "_file_mtime_scanned s, " BUILDIDS "_files f "
2895 12 : "where f.id = s.file");
2896 12 : sqlite_ps files_del_f_de (db, "nuke f_de", "delete from " BUILDIDS "_f_de where file = ? and mtime = ?");
2897 12 : sqlite_ps files_del_r_de (db, "nuke r_de", "delete from " BUILDIDS "_r_de where file = ? and mtime = ?");
2898 : sqlite_ps files_del_scan (db, "nuke f_m_s", "delete from " BUILDIDS "_file_mtime_scanned "
2899 12 : "where file = ? and mtime = ?");
2900 4 : files.reset();
2901 : while(1)
2902 : {
2903 40 : int rc = files.step();
2904 40 : if (rc != SQLITE_ROW)
2905 4 : break;
2906 :
2907 36 : int64_t mtime = sqlite3_column_int64 (files, 0);
2908 36 : int64_t fileid = sqlite3_column_int64 (files, 1);
2909 36 : const char* filename = ((const char*) sqlite3_column_text (files, 2) ?: "");
2910 : struct stat s;
2911 36 : rc = stat(filename, &s);
2912 36 : if (rc < 0 || (mtime != (int64_t) s.st_mtime))
2913 : {
2914 4 : if (verbose > 2)
2915 0 : obatched(clog) << "groom: forgetting file=" << filename << " mtime=" << mtime << endl;
2916 4 : files_del_f_de.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
2917 4 : files_del_r_de.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
2918 4 : files_del_scan.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
2919 : }
2920 36 : }
2921 4 : files.reset();
2922 :
2923 : // delete buildids with no references in _r_de or _f_de tables;
2924 : // cascades to _r_sref & _f_s records
2925 : sqlite_ps buildids_del (db, "nuke orphan buildids",
2926 : "delete from " BUILDIDS "_buildids "
2927 : "where not exists (select 1 from " BUILDIDS "_f_de d where " BUILDIDS "_buildids.id = d.buildid) "
2928 12 : "and not exists (select 1 from " BUILDIDS "_r_de d where " BUILDIDS "_buildids.id = d.buildid)");
2929 4 : buildids_del.reset().step_ok_done();
2930 :
2931 : // NB: "vacuum" is too heavy for even daily runs: it rewrites the entire db, so is done as maxigroom -G
2932 12 : sqlite_ps g1 (db, "incremental vacuum", "pragma incremental_vacuum");
2933 4 : g1.reset().step_ok_done();
2934 12 : sqlite_ps g2 (db, "optimize", "pragma optimize");
2935 4 : g2.reset().step_ok_done();
2936 8 : sqlite_ps g3 (db, "wal checkpoint", "pragma wal_checkpoint=truncate");
2937 4 : g3.reset().step_ok_done();
2938 :
2939 4 : database_stats_report();
2940 :
2941 4 : sqlite3_db_release_memory(db); // shrink the process if possible
2942 :
2943 4 : fdcache.limit(0,0); // release the fdcache contents
2944 4 : fdcache.limit(fdcache_fds,fdcache_mbs); // restore status quo parameters
2945 :
2946 4 : gettimeofday (&tv_end, NULL);
2947 4 : double deltas = (tv_end.tv_sec - tv_start.tv_sec) + (tv_end.tv_usec - tv_start.tv_usec)*0.000001;
2948 :
2949 4 : obatched(clog) << "groomed database in " << deltas << "s" << endl;
2950 4 : }
2951 :
2952 :
2953 : static void*
2954 3 : thread_main_groom (void* /*arg*/)
2955 : {
2956 3 : sig_atomic_t forced_groom_count = 0;
2957 3 : set_metric("thread_tid", "role", "groom", tid());
2958 3 : add_metric("thread_count", "role", "groom", 1);
2959 :
2960 3 : time_t last_groom = 0;
2961 :
2962 : while (1)
2963 : {
2964 13 : sleep (1);
2965 13 : scanq.wait_idle(); // PR25394: block scanners during grooming!
2966 13 : if (interrupted) break;
2967 :
2968 10 : time_t now = time(NULL);
2969 10 : bool groom_now = false;
2970 10 : if (last_groom == 0) // at least one initial groom is documented even for -g0
2971 3 : groom_now = true;
2972 10 : if (groom_s > 0 && (long)now > (long)(last_groom + groom_s))
2973 2 : groom_now = true;
2974 10 : if (sigusr2 != forced_groom_count)
2975 : {
2976 1 : forced_groom_count = sigusr2;
2977 1 : groom_now = true;
2978 : }
2979 10 : if (groom_now)
2980 : try
2981 : {
2982 4 : set_metric("thread_busy", "role", "groom", 1);
2983 4 : groom ();
2984 4 : last_groom = time(NULL); // NB: now was before grooming
2985 4 : inc_metric("thread_work_total", "role", "groom");
2986 4 : set_metric("thread_busy", "role", "groom", 0);
2987 : }
2988 0 : catch (const sqlite_exception& e)
2989 : {
2990 0 : obatched(cerr) << e.message << endl;
2991 : }
2992 :
2993 10 : scanq.done_idle();
2994 10 : }
2995 :
2996 3 : return 0;
2997 : }
2998 :
2999 :
3000 : ////////////////////////////////////////////////////////////////////////
3001 :
3002 :
3003 : static void
3004 3 : signal_handler (int /* sig */)
3005 : {
3006 3 : interrupted ++;
3007 :
3008 3 : if (db)
3009 3 : sqlite3_interrupt (db);
3010 :
3011 : // NB: don't do anything else in here
3012 3 : }
3013 :
3014 : static void
3015 4 : sigusr1_handler (int /* sig */)
3016 : {
3017 4 : sigusr1 ++;
3018 : // NB: don't do anything else in here
3019 4 : }
3020 :
3021 : static void
3022 1 : sigusr2_handler (int /* sig */)
3023 : {
3024 1 : sigusr2 ++;
3025 : // NB: don't do anything else in here
3026 1 : }
3027 :
3028 :
3029 :
3030 :
3031 :
3032 : // A user-defined sqlite function, to score the sharedness of the
3033 : // prefix of two strings. This is used to compare candidate debuginfo
3034 : // / source-rpm names, so that the closest match
3035 : // (directory-topology-wise closest) is found. This is important in
3036 : // case the same sref (source file name) is in many -debuginfo or
3037 : // -debugsource RPMs, such as when multiple versions/releases of the
3038 : // same package are in the database.
3039 :
3040 26 : static void sqlite3_sharedprefix_fn (sqlite3_context* c, int argc, sqlite3_value** argv)
3041 : {
3042 26 : if (argc != 2)
3043 0 : sqlite3_result_error(c, "expect 2 string arguments", -1);
3044 52 : else if ((sqlite3_value_type(argv[0]) != SQLITE_TEXT) ||
3045 26 : (sqlite3_value_type(argv[1]) != SQLITE_TEXT))
3046 4 : sqlite3_result_null(c);
3047 : else
3048 : {
3049 22 : const unsigned char* a = sqlite3_value_text (argv[0]);
3050 22 : const unsigned char* b = sqlite3_value_text (argv[1]);
3051 22 : int i = 0;
3052 1773 : while (*a++ == *b++)
3053 1751 : i++;
3054 22 : sqlite3_result_int (c, i);
3055 : }
3056 26 : }
3057 :
3058 :
3059 : int
3060 3 : main (int argc, char *argv[])
3061 : {
3062 3 : (void) setlocale (LC_ALL, "");
3063 3 : (void) bindtextdomain (PACKAGE_TARNAME, LOCALEDIR);
3064 3 : (void) textdomain (PACKAGE_TARNAME);
3065 :
3066 : /* Tell the library which version we are expecting. */
3067 3 : elf_version (EV_CURRENT);
3068 :
3069 3 : tmpdir = string(getenv("TMPDIR") ?: "/tmp");
3070 :
3071 : /* Set computed default values. */
3072 3 : db_path = string(getenv("HOME") ?: "/") + string("/.debuginfod.sqlite"); /* XDG? */
3073 3 : int rc = regcomp (& file_include_regex, ".*", REG_EXTENDED|REG_NOSUB); // match everything
3074 3 : if (rc != 0)
3075 : error (EXIT_FAILURE, 0, "regcomp failure: %d", rc);
3076 3 : rc = regcomp (& file_exclude_regex, "^$", REG_EXTENDED|REG_NOSUB); // match nothing
3077 3 : if (rc != 0)
3078 : error (EXIT_FAILURE, 0, "regcomp failure: %d", rc);
3079 :
3080 : // default parameters for fdcache are computed from system stats
3081 : struct statfs sfs;
3082 3 : rc = statfs(tmpdir.c_str(), &sfs);
3083 3 : if (rc < 0)
3084 0 : fdcache_mbs = 1024; // 1 gigabyte
3085 : else
3086 3 : fdcache_mbs = sfs.f_bavail * sfs.f_bsize / 1024 / 1024 / 4; // 25% of free space
3087 3 : fdcache_prefetch = 64; // guesstimate storage is this much less costly than re-decompression
3088 3 : fdcache_fds = (concurrency + fdcache_prefetch) * 2;
3089 :
3090 : /* Parse and process arguments. */
3091 : int remaining;
3092 3 : argp_program_version_hook = print_version; // this works
3093 3 : (void) argp_parse (&argp, argc, argv, ARGP_IN_ORDER, &remaining, NULL);
3094 3 : if (remaining != argc)
3095 0 : error (EXIT_FAILURE, 0,
3096 0 : "unexpected argument: %s", argv[remaining]);
3097 :
3098 3 : if (scan_archives.size()==0 && !scan_files && source_paths.size()>0)
3099 0 : obatched(clog) << "warning: without -F -R -U -Z, ignoring PATHs" << endl;
3100 :
3101 3 : fdcache.limit(fdcache_fds, fdcache_mbs);
3102 :
3103 3 : (void) signal (SIGPIPE, SIG_IGN); // microhttpd can generate it incidentally, ignore
3104 3 : (void) signal (SIGINT, signal_handler); // ^C
3105 3 : (void) signal (SIGHUP, signal_handler); // EOF
3106 3 : (void) signal (SIGTERM, signal_handler); // systemd
3107 3 : (void) signal (SIGUSR1, sigusr1_handler); // end-user
3108 3 : (void) signal (SIGUSR2, sigusr2_handler); // end-user
3109 :
3110 : /* Get database ready. */
3111 3 : rc = sqlite3_open_v2 (db_path.c_str(), &db, (SQLITE_OPEN_READWRITE
3112 : |SQLITE_OPEN_CREATE
3113 : |SQLITE_OPEN_FULLMUTEX), /* thread-safe */
3114 : NULL);
3115 3 : if (rc == SQLITE_CORRUPT)
3116 : {
3117 0 : (void) unlink (db_path.c_str());
3118 0 : error (EXIT_FAILURE, 0,
3119 : "cannot open %s, deleted database: %s", db_path.c_str(), sqlite3_errmsg(db));
3120 : }
3121 3 : else if (rc)
3122 : {
3123 0 : error (EXIT_FAILURE, 0,
3124 : "cannot open %s, consider deleting database: %s", db_path.c_str(), sqlite3_errmsg(db));
3125 : }
3126 :
3127 3 : obatched(clog) << "opened database " << db_path << endl;
3128 3 : obatched(clog) << "sqlite version " << sqlite3_version << endl;
3129 :
3130 : // add special string-prefix-similarity function used in rpm sref/sdef resolution
3131 3 : rc = sqlite3_create_function(db, "sharedprefix", 2, SQLITE_UTF8, NULL,
3132 : & sqlite3_sharedprefix_fn, NULL, NULL);
3133 3 : if (rc != SQLITE_OK)
3134 0 : error (EXIT_FAILURE, 0,
3135 : "cannot create sharedprefix( function: %s", sqlite3_errmsg(db));
3136 :
3137 3 : if (verbose > 3)
3138 0 : obatched(clog) << "ddl: " << DEBUGINFOD_SQLITE_DDL << endl;
3139 3 : rc = sqlite3_exec (db, DEBUGINFOD_SQLITE_DDL, NULL, NULL, NULL);
3140 3 : if (rc != SQLITE_OK)
3141 : {
3142 0 : error (EXIT_FAILURE, 0,
3143 : "cannot run database schema ddl: %s", sqlite3_errmsg(db));
3144 : }
3145 :
3146 : // Start httpd server threads. Separate pool for IPv4 and IPv6, in
3147 : // case the host only has one protocol stack.
3148 3 : MHD_Daemon *d4 = MHD_start_daemon (MHD_USE_THREAD_PER_CONNECTION
3149 : #if MHD_VERSION >= 0x00095300
3150 : | MHD_USE_INTERNAL_POLLING_THREAD
3151 : #else
3152 : | MHD_USE_SELECT_INTERNALLY
3153 : #endif
3154 : | MHD_USE_DEBUG, /* report errors to stderr */
3155 : http_port,
3156 : NULL, NULL, /* default accept policy */
3157 : handler_cb, NULL, /* handler callback */
3158 : MHD_OPTION_END);
3159 3 : MHD_Daemon *d6 = MHD_start_daemon (MHD_USE_THREAD_PER_CONNECTION
3160 : #if MHD_VERSION >= 0x00095300
3161 : | MHD_USE_INTERNAL_POLLING_THREAD
3162 : #else
3163 : | MHD_USE_SELECT_INTERNALLY
3164 : #endif
3165 : | MHD_USE_IPv6
3166 : | MHD_USE_DEBUG, /* report errors to stderr */
3167 : http_port,
3168 : NULL, NULL, /* default accept policy */
3169 : handler_cb, NULL, /* handler callback */
3170 : MHD_OPTION_END);
3171 :
3172 3 : if (d4 == NULL && d6 == NULL) // neither ipv4 nor ipv6? boo
3173 : {
3174 0 : sqlite3 *database = db;
3175 0 : db = 0; // for signal_handler not to freak
3176 0 : sqlite3_close (database);
3177 0 : error (EXIT_FAILURE, 0, "cannot start http server at port %d", http_port);
3178 : }
3179 :
3180 6 : obatched(clog) << "started http server on "
3181 : << (d4 != NULL ? "IPv4 " : "")
3182 : << (d6 != NULL ? "IPv6 " : "")
3183 3 : << "port=" << http_port << endl;
3184 :
3185 : // add maxigroom sql if -G given
3186 3 : if (maxigroom)
3187 : {
3188 0 : obatched(clog) << "maxigrooming database, please wait." << endl;
3189 0 : extra_ddl.push_back("create index if not exists " BUILDIDS "_r_sref_arc on " BUILDIDS "_r_sref(artifactsrc);");
3190 0 : extra_ddl.push_back("delete from " BUILDIDS "_r_sdef where not exists (select 1 from " BUILDIDS "_r_sref b where " BUILDIDS "_r_sdef.content = b.artifactsrc);");
3191 0 : extra_ddl.push_back("drop index if exists " BUILDIDS "_r_sref_arc;");
3192 :
3193 : // NB: we don't maxigroom the _files interning table. It'd require a temp index on all the
3194 : // tables that have file foreign-keys, which is a lot.
3195 :
3196 : // NB: with =delete, may take up 3x disk space total during vacuum process
3197 : // vs. =off (only 2x but may corrupt database if program dies mid-vacuum)
3198 : // vs. =wal (>3x observed, but safe)
3199 0 : extra_ddl.push_back("pragma journal_mode=delete;");
3200 0 : extra_ddl.push_back("vacuum;");
3201 0 : extra_ddl.push_back("pragma journal_mode=wal;");
3202 : }
3203 :
3204 : // run extra -D sql if given
3205 3 : for (auto&& i: extra_ddl)
3206 : {
3207 0 : if (verbose > 1)
3208 0 : obatched(clog) << "extra ddl:\n" << i << endl;
3209 0 : rc = sqlite3_exec (db, i.c_str(), NULL, NULL, NULL);
3210 0 : if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)
3211 0 : error (0, 0,
3212 : "warning: cannot run database extra ddl %s: %s", i.c_str(), sqlite3_errmsg(db));
3213 : }
3214 :
3215 3 : if (maxigroom)
3216 0 : obatched(clog) << "maxigroomed database" << endl;
3217 :
3218 3 : obatched(clog) << "search concurrency " << concurrency << endl;
3219 3 : obatched(clog) << "rescan time " << rescan_s << endl;
3220 3 : obatched(clog) << "fdcache fds " << fdcache_fds << endl;
3221 3 : obatched(clog) << "fdcache mbs " << fdcache_mbs << endl;
3222 3 : obatched(clog) << "fdcache prefetch " << fdcache_prefetch << endl;
3223 3 : obatched(clog) << "fdcache tmpdir " << tmpdir << endl;
3224 3 : obatched(clog) << "groom time " << groom_s << endl;
3225 3 : if (scan_archives.size()>0)
3226 : {
3227 6 : obatched ob(clog);
3228 3 : auto& o = ob << "scanning archive types ";
3229 10 : for (auto&& arch : scan_archives)
3230 7 : o << arch.first << "(" << arch.second << ") ";
3231 3 : o << endl;
3232 : }
3233 3 : const char* du = getenv(DEBUGINFOD_URLS_ENV_VAR);
3234 3 : if (du && du[0] != '\0') // set to non-empty string?
3235 2 : obatched(clog) << "upstream debuginfod servers: " << du << endl;
3236 :
3237 3 : vector<pthread_t> all_threads;
3238 :
3239 : pthread_t pt;
3240 3 : rc = pthread_create (& pt, NULL, thread_main_groom, NULL);
3241 3 : if (rc < 0)
3242 : error (0, 0, "warning: cannot spawn thread (%d) to groom database\n", rc);
3243 : else
3244 3 : all_threads.push_back(pt);
3245 :
3246 3 : if (scan_files || scan_archives.size() > 0)
3247 : {
3248 3 : pthread_create (& pt, NULL, thread_main_fts_source_paths, NULL);
3249 3 : if (rc < 0)
3250 : error (0, 0, "warning: cannot spawn thread (%d) to traverse source paths\n", rc);
3251 3 : all_threads.push_back(pt);
3252 15 : for (unsigned i=0; i<concurrency; i++)
3253 : {
3254 12 : pthread_create (& pt, NULL, thread_main_scanner, NULL);
3255 12 : if (rc < 0)
3256 : error (0, 0, "warning: cannot spawn thread (%d) to scan source files / archives\n", rc);
3257 12 : all_threads.push_back(pt);
3258 : }
3259 : }
3260 :
3261 : /* Trivial main loop! */
3262 3 : set_metric("ready", 1);
3263 11 : while (! interrupted)
3264 8 : pause ();
3265 3 : scanq.nuke(); // wake up any remaining scanq-related threads, let them die
3266 3 : set_metric("ready", 0);
3267 :
3268 3 : if (verbose)
3269 1 : obatched(clog) << "stopping" << endl;
3270 :
3271 : /* Join all our threads. */
3272 21 : for (auto&& it : all_threads)
3273 18 : pthread_join (it, NULL);
3274 :
3275 : /* Stop all the web service threads. */
3276 3 : if (d4) MHD_stop_daemon (d4);
3277 3 : if (d6) MHD_stop_daemon (d6);
3278 :
3279 : /* With all threads known dead, we can clean up the global resources. */
3280 3 : rc = sqlite3_exec (db, DEBUGINFOD_SQLITE_CLEANUP_DDL, NULL, NULL, NULL);
3281 3 : if (rc != SQLITE_OK)
3282 : {
3283 0 : error (0, 0,
3284 : "warning: cannot run database cleanup ddl: %s", sqlite3_errmsg(db));
3285 : }
3286 :
3287 : // NB: no problem with unconditional free here - an earlier failed regcomp would exit program
3288 3 : (void) regfree (& file_include_regex);
3289 3 : (void) regfree (& file_exclude_regex);
3290 :
3291 3 : sqlite3 *database = db;
3292 3 : db = 0; // for signal_handler not to freak
3293 3 : (void) sqlite3_close (database);
3294 :
3295 3 : return 0;
3296 : }
|
