tools/perf/util/bpf-loader.c

   1 /*
   2  * bpf-loader.c
   3  *
   4  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   5  * Copyright (C) 2015 Huawei Inc.
   6  */
   7
   8 #include <bpf/libbpf.h>
   9 #include <linux/err.h>
  10 #include <linux/string.h>
  11 #include "perf.h"
  12 #include "debug.h"
  13 #include "bpf-loader.h"
  14 #include "probe-event.h"
  15 #include "probe-finder.h" // for MAX_PROBES
  16 #include "llvm-utils.h"
  17
  18 #define DEFINE_PRINT_FN(name, level) \
  19 static int libbpf_##name(const char *fmt, ...)  \
  20 {                                               \
  21         va_list args;                           \
  22         int ret;                                \
  23                                                 \
  24         va_start(args, fmt);                    \
  25         ret = veprintf(level, verbose, pr_fmt(fmt), args);\
  26         va_end(args);                           \
  27         return ret;                             \
  28 }
  29
  30 DEFINE_PRINT_FN(warning, 1)
  31 DEFINE_PRINT_FN(info, 1)
  32 DEFINE_PRINT_FN(debug, 1)
  33
  34 struct bpf_prog_priv {
  35         struct perf_probe_event pev;
  36 };
  37
  38 static bool libbpf_initialized;
  39
  40 struct bpf_object *
  41 bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name)
  42 {
  43         struct bpf_object *obj;
  44
  45         if (!libbpf_initialized) {
  46                 libbpf_set_print(libbpf_warning,
  47                                  libbpf_info,
  48                                  libbpf_debug);
  49                 libbpf_initialized = true;
  50         }
  51
  52         obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name);
  53         if (IS_ERR(obj)) {
  54                 pr_debug("bpf: failed to load buffer\n");
  55                 return ERR_PTR(-EINVAL);
  56         }
  57
  58         return obj;
  59 }
  60
  61 struct bpf_object *bpf__prepare_load(const char *filename, bool source)
  62 {
  63         struct bpf_object *obj;
  64
  65         if (!libbpf_initialized) {
  66                 libbpf_set_print(libbpf_warning,
  67                                  libbpf_info,
  68                                  libbpf_debug);
  69                 libbpf_initialized = true;
  70         }
  71
  72         if (source) {
  73                 int err;
  74                 void *obj_buf;
  75                 size_t obj_buf_sz;
  76
  77                 err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz);
  78                 if (err)
  79                         return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE);
  80                 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename);
  81                 free(obj_buf);
  82         } else
  83                 obj = bpf_object__open(filename);
  84
  85         if (IS_ERR(obj)) {
  86                 pr_debug("bpf: failed to load %s\n", filename);
  87                 return obj;
  88         }
  89
  90         return obj;
  91 }
  92
  93 void bpf__clear(void)
  94 {
  95         struct bpf_object *obj, *tmp;
  96
  97         bpf_object__for_each_safe(obj, tmp) {
  98                 bpf__unprobe(obj);
  99                 bpf_object__close(obj);
 100         }
 101 }
 102
 103 static void
 104 bpf_prog_priv__clear(struct bpf_program *prog __maybe_unused,
 105                      void *_priv)
 106 {
 107         struct bpf_prog_priv *priv = _priv;
 108
 109         cleanup_perf_probe_events(&priv->pev, 1);
 110         free(priv);
 111 }
 112
 113 static int
 114 config__exec(const char *value, struct perf_probe_event *pev)
 115 {
 116         pev->uprobes = true;
 117         pev->target = strdup(value);
 118         if (!pev->target)
 119                 return -ENOMEM;
 120         return 0;
 121 }
 122
 123 static int
 124 config__module(const char *value, struct perf_probe_event *pev)
 125 {
 126         pev->uprobes = false;
 127         pev->target = strdup(value);
 128         if (!pev->target)
 129                 return -ENOMEM;
 130         return 0;
 131 }
 132
 133 static int
 134 config__bool(const char *value,
 135              bool *pbool, bool invert)
 136 {
 137         int err;
 138         bool bool_value;
 139
 140         if (!pbool)
 141                 return -EINVAL;
 142
 143         err = strtobool(value, &bool_value);
 144         if (err)
 145                 return err;
 146
 147         *pbool = invert ? !bool_value : bool_value;
 148         return 0;
 149 }
 150
 151 static int
 152 config__inlines(const char *value,
 153                 struct perf_probe_event *pev __maybe_unused)
 154 {
 155         return config__bool(value, &probe_conf.no_inlines, true);
 156 }
 157
 158 static int
 159 config__force(const char *value,
 160               struct perf_probe_event *pev __maybe_unused)
 161 {
 162         return config__bool(value, &probe_conf.force_add, false);
 163 }
 164
 165 static struct {
 166         const char *key;
 167         const char *usage;
 168         const char *desc;
 169         int (*func)(const char *, struct perf_probe_event *);
 170 } bpf_config_terms[] = {
 171         {
 172                 .key    = "exec",
 173                 .usage  = "exec=<full path of file>",
 174                 .desc   = "Set uprobe target",
 175                 .func   = config__exec,
 176         },
 177         {
 178                 .key    = "module",
 179                 .usage  = "module=<module name>    ",
 180                 .desc   = "Set kprobe module",
 181                 .func   = config__module,
 182         },
 183         {
 184                 .key    = "inlines",
 185                 .usage  = "inlines=[yes|no]        ",
 186                 .desc   = "Probe at inline symbol",
 187                 .func   = config__inlines,
 188         },
 189         {
 190                 .key    = "force",
 191                 .usage  = "force=[yes|no]          ",
 192                 .desc   = "Forcibly add events with existing name",
 193                 .func   = config__force,
 194         },
 195 };
 196
 197 static int
 198 do_config(const char *key, const char *value,
 199           struct perf_probe_event *pev)
 200 {
 201         unsigned int i;
 202
 203         pr_debug("config bpf program: %s=%s\n", key, value);
 204         for (i = 0; i < ARRAY_SIZE(bpf_config_terms); i++)
 205                 if (strcmp(key, bpf_config_terms[i].key) == 0)
 206                         return bpf_config_terms[i].func(value, pev);
 207
 208         pr_debug("BPF: ERROR: invalid config option in object: %s=%s\n",
 209                  key, value);
 210
 211         pr_debug("\nHint: Currently valid options are:\n");
 212         for (i = 0; i < ARRAY_SIZE(bpf_config_terms); i++)
 213                 pr_debug("\t%s:\t%s\n", bpf_config_terms[i].usage,
 214                          bpf_config_terms[i].desc);
 215         pr_debug("\n");
 216
 217         return -BPF_LOADER_ERRNO__CONFIG_TERM;
 218 }
 219
 220 static const char *
 221 parse_config_kvpair(const char *config_str, struct perf_probe_event *pev)
 222 {
 223         char *text = strdup(config_str);
 224         char *sep, *line;
 225         const char *main_str = NULL;
 226         int err = 0;
 227
 228         if (!text) {
 229                 pr_debug("No enough memory: dup config_str failed\n");
 230                 return ERR_PTR(-ENOMEM);
 231         }
 232
 233         line = text;
 234         while ((sep = strchr(line, ';'))) {
 235                 char *equ;
 236
 237                 *sep = '\0';
 238                 equ = strchr(line, '=');
 239                 if (!equ) {
 240                         pr_warning("WARNING: invalid config in BPF object: %s\n",
 241                                    line);
 242                         pr_warning("\tShould be 'key=value'.\n");
 243                         goto nextline;
 244                 }
 245                 *equ = '\0';
 246
 247                 err = do_config(line, equ + 1, pev);
 248                 if (err)
 249                         break;
 250 nextline:
 251                 line = sep + 1;
 252         }
 253
 254         if (!err)
 255                 main_str = config_str + (line - text);
 256         free(text);
 257
 258         return err ? ERR_PTR(err) : main_str;
 259 }
 260
 261 static int
 262 parse_config(const char *config_str, struct perf_probe_event *pev)
 263 {
 264         int err;
 265         const char *main_str = parse_config_kvpair(config_str, pev);
 266
 267         if (IS_ERR(main_str))
 268                 return PTR_ERR(main_str);
 269
 270         err = parse_perf_probe_command(main_str, pev);
 271         if (err < 0) {
 272                 pr_debug("bpf: '%s' is not a valid config string\n",
 273                          config_str);
 274                 /* parse failed, don't need clear pev. */
 275                 return -BPF_LOADER_ERRNO__CONFIG;
 276         }
 277         return 0;
 278 }
 279
 280 static int
 281 config_bpf_program(struct bpf_program *prog)
 282 {
 283         struct perf_probe_event *pev = NULL;
 284         struct bpf_prog_priv *priv = NULL;
 285         const char *config_str;
 286         int err;
 287
 288         /* Initialize per-program probing setting */
 289         probe_conf.no_inlines = false;
 290         probe_conf.force_add = false;
 291
 292         config_str = bpf_program__title(prog, false);
 293         if (IS_ERR(config_str)) {
 294                 pr_debug("bpf: unable to get title for program\n");
 295                 return PTR_ERR(config_str);
 296         }
 297
 298         priv = calloc(sizeof(*priv), 1);
 299         if (!priv) {
 300                 pr_debug("bpf: failed to alloc priv\n");
 301                 return -ENOMEM;
 302         }
 303         pev = &priv->pev;
 304
 305         pr_debug("bpf: config program '%s'\n", config_str);
 306         err = parse_config(config_str, pev);
 307         if (err)
 308                 goto errout;
 309
 310         if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) {
 311                 pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
 312                          config_str, PERF_BPF_PROBE_GROUP);
 313                 err = -BPF_LOADER_ERRNO__GROUP;
 314                 goto errout;
 315         } else if (!pev->group)
 316                 pev->group = strdup(PERF_BPF_PROBE_GROUP);
 317
 318         if (!pev->group) {
 319                 pr_debug("bpf: strdup failed\n");
 320                 err = -ENOMEM;
 321                 goto errout;
 322         }
 323
 324         if (!pev->event) {
 325                 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
 326                          config_str);
 327                 err = -BPF_LOADER_ERRNO__EVENTNAME;
 328                 goto errout;
 329         }
 330         pr_debug("bpf: config '%s' is ok\n", config_str);
 331
 332         err = bpf_program__set_private(prog, priv, bpf_prog_priv__clear);
 333         if (err) {
 334                 pr_debug("Failed to set priv for program '%s'\n", config_str);
 335                 goto errout;
 336         }
 337
 338         return 0;
 339
 340 errout:
 341         if (pev)
 342                 clear_perf_probe_event(pev);
 343         free(priv);
 344         return err;
 345 }
 346
 347 static int bpf__prepare_probe(void)
 348 {
 349         static int err = 0;
 350         static bool initialized = false;
 351
 352         /*
 353          * Make err static, so if init failed the first, bpf__prepare_probe()
 354          * fails each time without calling init_probe_symbol_maps multiple
 355          * times.
 356          */
 357         if (initialized)
 358                 return err;
 359
 360         initialized = true;
 361         err = init_probe_symbol_maps(false);
 362         if (err < 0)
 363                 pr_debug("Failed to init_probe_symbol_maps\n");
 364         probe_conf.max_probes = MAX_PROBES;
 365         return err;
 366 }
 367
 368 int bpf__probe(struct bpf_object *obj)
 369 {
 370         int err = 0;
 371         struct bpf_program *prog;
 372         struct bpf_prog_priv *priv;
 373         struct perf_probe_event *pev;
 374
 375         err = bpf__prepare_probe();
 376         if (err) {
 377                 pr_debug("bpf__prepare_probe failed\n");
 378                 return err;
 379         }
 380
 381         bpf_object__for_each_program(prog, obj) {
 382                 err = config_bpf_program(prog);
 383                 if (err)
 384                         goto out;
 385
 386                 err = bpf_program__get_private(prog, (void **)&priv);
 387                 if (err || !priv)
 388                         goto out;
 389                 pev = &priv->pev;
 390
 391                 err = convert_perf_probe_events(pev, 1);
 392                 if (err < 0) {
 393                         pr_debug("bpf_probe: failed to convert perf probe events");
 394                         goto out;
 395                 }
 396
 397                 err = apply_perf_probe_events(pev, 1);
 398                 if (err < 0) {
 399                         pr_debug("bpf_probe: failed to apply perf probe events");
 400                         goto out;
 401                 }
 402         }
 403 out:
 404         return err < 0 ? err : 0;
 405 }
 406
 407 #define EVENTS_WRITE_BUFSIZE  4096
 408 int bpf__unprobe(struct bpf_object *obj)
 409 {
 410         int err, ret = 0;
 411         struct bpf_program *prog;
 412         struct bpf_prog_priv *priv;
 413
 414         bpf_object__for_each_program(prog, obj) {
 415                 int i;
 416
 417                 err = bpf_program__get_private(prog, (void **)&priv);
 418                 if (err || !priv)
 419                         continue;
 420
 421                 for (i = 0; i < priv->pev.ntevs; i++) {
 422                         struct probe_trace_event *tev = &priv->pev.tevs[i];
 423                         char name_buf[EVENTS_WRITE_BUFSIZE];
 424                         struct strfilter *delfilter;
 425
 426                         snprintf(name_buf, EVENTS_WRITE_BUFSIZE,
 427                                  "%s:%s", tev->group, tev->event);
 428                         name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0';
 429
 430                         delfilter = strfilter__new(name_buf, NULL);
 431                         if (!delfilter) {
 432                                 pr_debug("Failed to create filter for unprobing\n");
 433                                 ret = -ENOMEM;
 434                                 continue;
 435                         }
 436
 437                         err = del_perf_probe_events(delfilter);
 438                         strfilter__delete(delfilter);
 439                         if (err) {
 440                                 pr_debug("Failed to delete %s\n", name_buf);
 441                                 ret = err;
 442                                 continue;
 443                         }
 444                 }
 445         }
 446         return ret;
 447 }
 448
 449 int bpf__load(struct bpf_object *obj)
 450 {
 451         int err;
 452
 453         err = bpf_object__load(obj);
 454         if (err) {
 455                 pr_debug("bpf: load objects failed\n");
 456                 return err;
 457         }
 458         return 0;
 459 }
 460
 461 int bpf__foreach_tev(struct bpf_object *obj,
 462                      bpf_prog_iter_callback_t func,
 463                      void *arg)
 464 {
 465         struct bpf_program *prog;
 466         int err;
 467
 468         bpf_object__for_each_program(prog, obj) {
 469                 struct probe_trace_event *tev;
 470                 struct perf_probe_event *pev;
 471                 struct bpf_prog_priv *priv;
 472                 int i, fd;
 473
 474                 err = bpf_program__get_private(prog,
 475                                 (void **)&priv);
 476                 if (err || !priv) {
 477                         pr_debug("bpf: failed to get private field\n");
 478                         return -BPF_LOADER_ERRNO__INTERNAL;
 479                 }
 480
 481                 pev = &priv->pev;
 482                 for (i = 0; i < pev->ntevs; i++) {
 483                         tev = &pev->tevs[i];
 484
 485                         fd = bpf_program__fd(prog);
 486                         if (fd < 0) {
 487                                 pr_debug("bpf: failed to get file descriptor\n");
 488                                 return fd;
 489                         }
 490
 491                         err = (*func)(tev, fd, arg);
 492                         if (err) {
 493                                 pr_debug("bpf: call back failed, stop iterate\n");
 494                                 return err;
 495                         }
 496                 }
 497         }
 498         return 0;
 499 }
 500
 501 #define ERRNO_OFFSET(e)         ((e) - __BPF_LOADER_ERRNO__START)
 502 #define ERRCODE_OFFSET(c)       ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
 503 #define NR_ERRNO        (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
 504
 505 static const char *bpf_loader_strerror_table[NR_ERRNO] = {
 506         [ERRCODE_OFFSET(CONFIG)]        = "Invalid config string",
 507         [ERRCODE_OFFSET(GROUP)]         = "Invalid group name",
 508         [ERRCODE_OFFSET(EVENTNAME)]     = "No event name found in config string",
 509         [ERRCODE_OFFSET(INTERNAL)]      = "BPF loader internal error",
 510         [ERRCODE_OFFSET(COMPILE)]       = "Error when compiling BPF scriptlet",
 511         [ERRCODE_OFFSET(CONFIG_TERM)]   = "Invalid config term in config string",
 512         [ERRCODE_OFFSET(PROLOGUE)]      = "Failed to generate prologue",
 513         [ERRCODE_OFFSET(PROLOGUE2BIG)]  = "Prologue too big for program",
 514         [ERRCODE_OFFSET(PROLOGUEOOB)]   = "Offset out of bound for prologue",
 515 };
 516
 517 static int
 518 bpf_loader_strerror(int err, char *buf, size_t size)
 519 {
 520         char sbuf[STRERR_BUFSIZE];
 521         const char *msg;
 522
 523         if (!buf || !size)
 524                 return -1;
 525
 526         err = err > 0 ? err : -err;
 527
 528         if (err >= __LIBBPF_ERRNO__START)
 529                 return libbpf_strerror(err, buf, size);
 530
 531         if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) {
 532                 msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)];
 533                 snprintf(buf, size, "%s", msg);
 534                 buf[size - 1] = '\0';
 535                 return 0;
 536         }
 537
 538         if (err >= __BPF_LOADER_ERRNO__END)
 539                 snprintf(buf, size, "Unknown bpf loader error %d", err);
 540         else
 541                 snprintf(buf, size, "%s",
 542                          strerror_r(err, sbuf, sizeof(sbuf)));
 543
 544         buf[size - 1] = '\0';
 545         return -1;
 546 }
 547
 548 #define bpf__strerror_head(err, buf, size) \
 549         char sbuf[STRERR_BUFSIZE], *emsg;\
 550         if (!size)\
 551                 return 0;\
 552         if (err < 0)\
 553                 err = -err;\
 554         bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
 555         emsg = sbuf;\
 556         switch (err) {\
 557         default:\
 558                 scnprintf(buf, size, "%s", emsg);\
 559                 break;
 560
 561 #define bpf__strerror_entry(val, fmt...)\
 562         case val: {\
 563                 scnprintf(buf, size, fmt);\
 564                 break;\
 565         }
 566
 567 #define bpf__strerror_end(buf, size)\
 568         }\
 569         buf[size - 1] = '\0';
 570
 571 int bpf__strerror_prepare_load(const char *filename, bool source,
 572                                int err, char *buf, size_t size)
 573 {
 574         size_t n;
 575         int ret;
 576
 577         n = snprintf(buf, size, "Failed to load %s%s: ",
 578                          filename, source ? " from source" : "");
 579         if (n >= size) {
 580                 buf[size - 1] = '\0';
 581                 return 0;
 582         }
 583         buf += n;
 584         size -= n;
 585
 586         ret = bpf_loader_strerror(err, buf, size);
 587         buf[size - 1] = '\0';
 588         return ret;
 589 }
 590
 591 int bpf__strerror_probe(struct bpf_object *obj __maybe_unused,
 592                         int err, char *buf, size_t size)
 593 {
 594         bpf__strerror_head(err, buf, size);
 595         case BPF_LOADER_ERRNO__CONFIG_TERM: {
 596                 scnprintf(buf, size, "%s (add -v to see detail)", emsg);
 597                 break;
 598         }
 599         bpf__strerror_entry(EEXIST, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'");
 600         bpf__strerror_entry(EACCES, "You need to be root");
 601         bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
 602         bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file");
 603         bpf__strerror_end(buf, size);
 604         return 0;
 605 }
 606
 607 int bpf__strerror_load(struct bpf_object *obj,
 608                        int err, char *buf, size_t size)
 609 {
 610         bpf__strerror_head(err, buf, size);
 611         case LIBBPF_ERRNO__KVER: {
 612                 unsigned int obj_kver = bpf_object__get_kversion(obj);
 613                 unsigned int real_kver;
 614
 615                 if (fetch_kernel_version(&real_kver, NULL, 0)) {
 616                         scnprintf(buf, size, "Unable to fetch kernel version");
 617                         break;
 618                 }
 619
 620                 if (obj_kver != real_kver) {
 621                         scnprintf(buf, size,
 622                                   "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")",
 623                                   KVER_PARAM(obj_kver),
 624                                   KVER_PARAM(real_kver));
 625                         break;
 626                 }
 627
 628                 scnprintf(buf, size, "Failed to load program for unknown reason");
 629                 break;
 630         }
 631         bpf__strerror_end(buf, size);
 632         return 0;
 633 }