src/bin/lttng-sessiond/kernel.c

   1 /*
   2  * Copyright (C) 2011 David Goulet <david.goulet@polymtl.ca>
   3  *
   4  * SPDX-License-Identifier: GPL-2.0-only
   5  *
   6  */
   7
   8 #define _LGPL_SOURCE
   9 #include <fcntl.h>
  10 #include <stdlib.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <unistd.h>
  14 #include <inttypes.h>
  15 #include <sys/types.h>
  16
  17 #include <common/common.h>
  18 #include <common/hashtable/utils.h>
  19 #include <common/trace-chunk.h>
  20 #include <common/kernel-ctl/kernel-ctl.h>
  21 #include <common/kernel-ctl/kernel-ioctl.h>
  22 #include <common/sessiond-comm/sessiond-comm.h>
  23 #include <common/tracker.h>
  24 #include <common/utils.h>
  25 #include <lttng/event.h>
  26 #include <lttng/lttng-error.h>
  27 #include <lttng/tracker.h>
  28
  29 #include <lttng/userspace-probe.h>
  30 #include <lttng/userspace-probe-internal.h>
  31 #include <lttng/condition/on-event.h>
  32 #include <lttng/condition/on-event-internal.h>
  33 #include <lttng/event-rule/event-rule.h>
  34 #include <lttng/event-rule/event-rule-internal.h>
  35 #include <lttng/event-rule/userspace-probe-internal.h>
  36
  37 #include "event-notifier-error-accounting.h"
  38 #include "lttng-sessiond.h"
  39 #include "lttng-syscall.h"
  40 #include "condition-internal.h"
  41 #include "consumer.h"
  42 #include "kernel.h"
  43 #include "kernel-consumer.h"
  44 #include "kern-modules.h"
  45 #include "sessiond-config.h"
  46 #include "utils.h"
  47 #include "rotate.h"
  48 #include "modprobe.h"
  49 #include "tracker.h"
  50 #include "notification-thread-commands.h"
  51
  52 /*
  53  * Key used to reference a channel between the sessiond and the consumer. This
  54  * is only read and updated with the session_list lock held.
  55  */
  56 static uint64_t next_kernel_channel_key;
  57
  58 static const char *module_proc_lttng = "/proc/lttng";
  59
  60 static int kernel_tracer_fd = -1;
  61 static int kernel_tracer_event_notifier_group_fd = -1;
  62 static int kernel_tracer_event_notifier_group_notification_fd = -1;
  63 static struct cds_lfht *kernel_token_to_event_notifier_rule_ht;
  64
  65 /*
  66  * Add context on a kernel channel.
  67  *
  68  * Assumes the ownership of ctx.
  69  */
  70 int kernel_add_channel_context(struct ltt_kernel_channel *chan,
  71                 struct ltt_kernel_context *ctx)
  72 {
  73         int ret;
  74
  75         assert(chan);
  76         assert(ctx);
  77
  78         DBG("Adding context to channel %s", chan->channel->name);
  79         ret = kernctl_add_context(chan->fd, &ctx->ctx);
  80         if (ret < 0) {
  81                 switch (-ret) {
  82                 case ENOSYS:
  83                         /* Exists but not available for this kernel */
  84                         ret = LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE;
  85                         goto error;
  86                 case EEXIST:
  87                         /* If EEXIST, we just ignore the error */
  88                         ret = 0;
  89                         goto end;
  90                 default:
  91                         PERROR("add context ioctl");
  92                         ret = LTTNG_ERR_KERN_CONTEXT_FAIL;
  93                         goto error;
  94                 }
  95         }
  96         ret = 0;
  97
  98 end:
  99         cds_list_add_tail(&ctx->list, &chan->ctx_list);
 100         ctx->in_list = true;
 101         ctx = NULL;
 102 error:
 103         if (ctx) {
 104                 trace_kernel_destroy_context(ctx);
 105         }
 106         return ret;
 107 }
 108
 109 /*
 110  * Create a new kernel session, register it to the kernel tracer and add it to
 111  * the session daemon session.
 112  */
 113 int kernel_create_session(struct ltt_session *session)
 114 {
 115         int ret;
 116         struct ltt_kernel_session *lks;
 117
 118         assert(session);
 119
 120         /* Allocate data structure */
 121         lks = trace_kernel_create_session();
 122         if (lks == NULL) {
 123                 ret = -1;
 124                 goto error;
 125         }
 126
 127         /* Kernel tracer session creation */
 128         ret = kernctl_create_session(kernel_tracer_fd);
 129         if (ret < 0) {
 130                 PERROR("ioctl kernel create session");
 131                 goto error;
 132         }
 133
 134         lks->fd = ret;
 135         /* Prevent fd duplication after execlp() */
 136         ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
 137         if (ret < 0) {
 138                 PERROR("fcntl session fd");
 139         }
 140
 141         lks->id = session->id;
 142         lks->consumer_fds_sent = 0;
 143         session->kernel_session = lks;
 144
 145         DBG("Kernel session created (fd: %d)", lks->fd);
 146
 147         /*
 148          * This is necessary since the creation time is present in the session
 149          * name when it is generated.
 150          */
 151         if (session->has_auto_generated_name) {
 152                 ret = kernctl_session_set_name(lks->fd, DEFAULT_SESSION_NAME);
 153         } else {
 154                 ret = kernctl_session_set_name(lks->fd, session->name);
 155         }
 156         if (ret) {
 157                 WARN("Could not set kernel session name for session %" PRIu64 " name: %s",
 158                         session->id, session->name);
 159         }
 160
 161         ret = kernctl_session_set_creation_time(lks->fd, session->creation_time);
 162         if (ret) {
 163                 WARN("Could not set kernel session creation time for session %" PRIu64 " name: %s",
 164                         session->id, session->name);
 165         }
 166
 167         return 0;
 168
 169 error:
 170         if (lks) {
 171                 trace_kernel_destroy_session(lks);
 172                 trace_kernel_free_session(lks);
 173         }
 174         return ret;
 175 }
 176
 177 /*
 178  * Create a kernel channel, register it to the kernel tracer and add it to the
 179  * kernel session.
 180  */
 181 int kernel_create_channel(struct ltt_kernel_session *session,
 182                 struct lttng_channel *chan)
 183 {
 184         int ret;
 185         struct ltt_kernel_channel *lkc;
 186
 187         assert(session);
 188         assert(chan);
 189
 190         /* Allocate kernel channel */
 191         lkc = trace_kernel_create_channel(chan);
 192         if (lkc == NULL) {
 193                 goto error;
 194         }
 195
 196         DBG3("Kernel create channel %s with attr: %d, %" PRIu64 ", %" PRIu64 ", %u, %u, %d, %d",
 197                         chan->name, lkc->channel->attr.overwrite,
 198                         lkc->channel->attr.subbuf_size, lkc->channel->attr.num_subbuf,
 199                         lkc->channel->attr.switch_timer_interval, lkc->channel->attr.read_timer_interval,
 200                         lkc->channel->attr.live_timer_interval, lkc->channel->attr.output);
 201
 202         /* Kernel tracer channel creation */
 203         ret = kernctl_create_channel(session->fd, &lkc->channel->attr);
 204         if (ret < 0) {
 205                 PERROR("ioctl kernel create channel");
 206                 goto error;
 207         }
 208
 209         /* Setup the channel fd */
 210         lkc->fd = ret;
 211         /* Prevent fd duplication after execlp() */
 212         ret = fcntl(lkc->fd, F_SETFD, FD_CLOEXEC);
 213         if (ret < 0) {
 214                 PERROR("fcntl session fd");
 215         }
 216
 217         /* Add channel to session */
 218         cds_list_add(&lkc->list, &session->channel_list.head);
 219         session->channel_count++;
 220         lkc->session = session;
 221         lkc->key = ++next_kernel_channel_key;
 222
 223         DBG("Kernel channel %s created (fd: %d, key: %" PRIu64 ")",
 224                         lkc->channel->name, lkc->fd, lkc->key);
 225
 226         return 0;
 227
 228 error:
 229         if (lkc) {
 230                 free(lkc->channel);
 231                 free(lkc);
 232         }
 233         return -1;
 234 }
 235
 236 /*
 237  * Create a kernel event notifier group, register it to the kernel tracer and
 238  * add it to the kernel session.
 239  */
 240 static int kernel_create_event_notifier_group(int *event_notifier_group_fd)
 241 {
 242         int ret;
 243         int local_fd = -1;
 244
 245         assert(event_notifier_group_fd);
 246
 247         /* Kernel event notifier group creation. */
 248         ret = kernctl_create_event_notifier_group(kernel_tracer_fd);
 249         if (ret < 0) {
 250                 PERROR("Failed to create kernel event notifier group");
 251                 ret = -1;
 252                 goto error;
 253         }
 254
 255         local_fd = ret;
 256
 257         /* Prevent fd duplication after execlp(). */
 258         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
 259         if (ret < 0) {
 260                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group file descriptor: fd = %d",
 261                                 local_fd);
 262                 goto error;
 263         }
 264
 265         DBG("Created kernel event notifier group: fd = %d", local_fd);
 266         *event_notifier_group_fd = local_fd;
 267         local_fd = -1;
 268         ret = 0;
 269 error:
 270         if (local_fd >= 0) {
 271                 ret = close(local_fd);
 272                 if (ret) {
 273                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
 274                                         local_fd);
 275                 }
 276         }
 277
 278         return ret;
 279 }
 280
 281 /*
 282  * Compute the offset of the instrumentation byte in the binary based on the
 283  * function probe location using the ELF lookup method.
 284  *
 285  * Returns 0 on success and set the offset out parameter to the offset of the
 286  * elf symbol
 287  * Returns -1 on error
 288  */
 289 static
 290 int extract_userspace_probe_offset_function_elf(
 291                 const struct lttng_userspace_probe_location *probe_location,
 292                 uid_t uid, gid_t gid, uint64_t *offset)
 293 {
 294         int fd;
 295         int ret = 0;
 296         const char *symbol = NULL;
 297         const struct lttng_userspace_probe_location_lookup_method *lookup = NULL;
 298         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 299
 300         assert(lttng_userspace_probe_location_get_type(probe_location) ==
 301                         LTTNG_USERSPACE_PROBE_LOCATION_TYPE_FUNCTION);
 302
 303         lookup = lttng_userspace_probe_location_get_lookup_method(
 304                         probe_location);
 305         if (!lookup) {
 306                 ret = -1;
 307                 goto end;
 308         }
 309
 310         lookup_method_type =
 311                         lttng_userspace_probe_location_lookup_method_get_type(lookup);
 312
 313         assert(lookup_method_type ==
 314                         LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF);
 315
 316         symbol = lttng_userspace_probe_location_function_get_function_name(
 317                         probe_location);
 318         if (!symbol) {
 319                 ret = -1;
 320                 goto end;
 321         }
 322
 323         fd = lttng_userspace_probe_location_function_get_binary_fd(probe_location);
 324         if (fd < 0) {
 325                 ret = -1;
 326                 goto end;
 327         }
 328
 329         ret = run_as_extract_elf_symbol_offset(fd, symbol, uid, gid, offset);
 330         if (ret < 0) {
 331                 DBG("userspace probe offset calculation failed for "
 332                                 "function %s", symbol);
 333                 goto end;
 334         }
 335
 336         DBG("userspace probe elf offset for %s is 0x%jd", symbol, (intmax_t)(*offset));
 337 end:
 338         return ret;
 339 }
 340
 341 /*
 342  * Compute the offsets of the instrumentation bytes in the binary based on the
 343  * tracepoint probe location using the SDT lookup method. This function
 344  * allocates the offsets buffer, the caller must free it.
 345  *
 346  * Returns 0 on success and set the offset out parameter to the offsets of the
 347  * SDT tracepoint.
 348  * Returns -1 on error.
 349  */
 350 static
 351 int extract_userspace_probe_offset_tracepoint_sdt(
 352                 const struct lttng_userspace_probe_location *probe_location,
 353                 uid_t uid, gid_t gid, uint64_t **offsets,
 354                 uint32_t *offsets_count)
 355 {
 356         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 357         const struct lttng_userspace_probe_location_lookup_method *lookup = NULL;
 358         const char *probe_name = NULL, *provider_name = NULL;
 359         int ret = 0;
 360         int fd, i;
 361
 362         assert(lttng_userspace_probe_location_get_type(probe_location) ==
 363                         LTTNG_USERSPACE_PROBE_LOCATION_TYPE_TRACEPOINT);
 364
 365         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 366         if (!lookup) {
 367                 ret = -1;
 368                 goto end;
 369         }
 370
 371         lookup_method_type =
 372                         lttng_userspace_probe_location_lookup_method_get_type(lookup);
 373
 374         assert(lookup_method_type ==
 375                         LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT);
 376
 377
 378         probe_name = lttng_userspace_probe_location_tracepoint_get_probe_name(
 379                         probe_location);
 380         if (!probe_name) {
 381                 ret = -1;
 382                 goto end;
 383         }
 384
 385         provider_name = lttng_userspace_probe_location_tracepoint_get_provider_name(
 386                         probe_location);
 387         if (!provider_name) {
 388                 ret = -1;
 389                 goto end;
 390         }
 391
 392         fd = lttng_userspace_probe_location_tracepoint_get_binary_fd(probe_location);
 393         if (fd < 0) {
 394                 ret = -1;
 395                 goto end;
 396         }
 397
 398         ret = run_as_extract_sdt_probe_offsets(fd, provider_name, probe_name,
 399                         uid, gid, offsets, offsets_count);
 400         if (ret < 0) {
 401                 DBG("userspace probe offset calculation failed for sdt "
 402                                 "probe %s:%s", provider_name, probe_name);
 403                 goto end;
 404         }
 405
 406         if (*offsets_count == 0) {
 407                 DBG("no userspace probe offset found");
 408                 goto end;
 409         }
 410
 411         DBG("%u userspace probe SDT offsets found for %s:%s at:",
 412                         *offsets_count, provider_name, probe_name);
 413         for (i = 0; i < *offsets_count; i++) {
 414                 DBG("\t0x%jd", (intmax_t)((*offsets)[i]));
 415         }
 416 end:
 417         return ret;
 418 }
 419
 420 static
 421 int userspace_probe_add_callsite(
 422                 const struct lttng_userspace_probe_location *location,
 423                 uid_t uid, gid_t gid, int fd)
 424 {
 425         const struct lttng_userspace_probe_location_lookup_method *lookup_method = NULL;
 426         enum lttng_userspace_probe_location_lookup_method_type type;
 427         int ret;
 428
 429         lookup_method = lttng_userspace_probe_location_get_lookup_method(location);
 430         if (!lookup_method) {
 431                 ret = -1;
 432                 goto end;
 433         }
 434
 435         type = lttng_userspace_probe_location_lookup_method_get_type(lookup_method);
 436         switch (type) {
 437         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF:
 438         {
 439                 struct lttng_kernel_event_callsite callsite;
 440                 uint64_t offset;
 441
 442                 ret = extract_userspace_probe_offset_function_elf(location,
 443                                 uid, gid, &offset);
 444                 if (ret) {
 445                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 446                         goto end;
 447                 }
 448
 449                 callsite.u.uprobe.offset = offset;
 450                 ret = kernctl_add_callsite(fd, &callsite);
 451                 if (ret) {
 452                         WARN("Failed to add callsite to ELF userspace probe.");
 453                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 454                         goto end;
 455                 }
 456                 break;
 457         }
 458         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT:
 459         {
 460                 int i;
 461                 uint64_t *offsets = NULL;
 462                 uint32_t offsets_count;
 463                 struct lttng_kernel_event_callsite callsite;
 464
 465                 /*
 466                  * This call allocates the offsets buffer. This buffer must be freed
 467                  * by the caller
 468                  */
 469                 ret = extract_userspace_probe_offset_tracepoint_sdt(location,
 470                                 uid, gid, &offsets, &offsets_count);
 471                 if (ret) {
 472                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 473                         goto end;
 474                 }
 475                 for (i = 0; i < offsets_count; i++) {
 476                         callsite.u.uprobe.offset = offsets[i];
 477                         ret = kernctl_add_callsite(fd, &callsite);
 478                         if (ret) {
 479                                 WARN("Failed to add callsite to SDT userspace probe");
 480                                 ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 481                                 free(offsets);
 482                                 goto end;
 483                         }
 484                 }
 485                 free(offsets);
 486                 break;
 487         }
 488         default:
 489                 ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 490                 goto end;
 491         }
 492 end:
 493         return ret;
 494 }
 495
 496 /*
 497  * Extract the offsets of the instrumentation point for the different lookup
 498  * methods.
 499  */
 500 static
 501 int userspace_probe_event_add_callsites(struct lttng_event *ev,
 502                         struct ltt_kernel_session *session, int fd)
 503 {
 504         int ret;
 505         const struct lttng_userspace_probe_location *location = NULL;
 506
 507         assert(ev);
 508         assert(ev->type == LTTNG_EVENT_USERSPACE_PROBE);
 509
 510         location = lttng_event_get_userspace_probe_location(ev);
 511         if (!location) {
 512                 ret = -1;
 513                 goto end;
 514         }
 515
 516         ret = userspace_probe_add_callsite(location, session->uid, session->gid,
 517                 fd);
 518         if (ret) {
 519                 WARN("Failed to add callsite to userspace probe event '%s'",
 520                                 ev->name);
 521         }
 522
 523 end:
 524         return ret;
 525 }
 526
 527 /*
 528  * Extract the offsets of the instrumentation point for the different look-up
 529  * methods.
 530  */
 531 static int userspace_probe_event_rule_add_callsites(
 532                 const struct lttng_event_rule *rule,
 533                 const struct lttng_credentials *creds,
 534                 int fd)
 535 {
 536         int ret;
 537         enum lttng_event_rule_status status;
 538         enum lttng_event_rule_type event_rule_type;
 539         const struct lttng_userspace_probe_location *location = NULL;
 540
 541         assert(rule);
 542         assert(creds);
 543
 544         event_rule_type = lttng_event_rule_get_type(rule);
 545         assert(event_rule_type == LTTNG_EVENT_RULE_TYPE_USERSPACE_PROBE);
 546
 547         status = lttng_event_rule_userspace_probe_get_location(rule, &location);
 548         if (status != LTTNG_EVENT_RULE_STATUS_OK || !location) {
 549                 ret = -1;
 550                 goto end;
 551         }
 552
 553         ret = userspace_probe_add_callsite(location,
 554                         lttng_credentials_get_uid(creds),
 555                         lttng_credentials_get_gid(creds), fd);
 556         if (ret) {
 557                 WARN("Failed to add callsite to user space probe object: fd = %d",
 558                                 fd);
 559         }
 560
 561 end:
 562         return ret;
 563 }
 564
 565 /*
 566  * Create a kernel event, enable it to the kernel tracer and add it to the
 567  * channel event list of the kernel session.
 568  * We own filter_expression and filter.
 569  */
 570 int kernel_create_event(struct lttng_event *ev,
 571                 struct ltt_kernel_channel *channel,
 572                 char *filter_expression,
 573                 struct lttng_bytecode *filter)
 574 {
 575         int err, fd;
 576         enum lttng_error_code ret;
 577         struct ltt_kernel_event *event;
 578
 579         assert(ev);
 580         assert(channel);
 581
 582         /* We pass ownership of filter_expression and filter */
 583         ret = trace_kernel_create_event(ev, filter_expression,
 584                         filter, &event);
 585         if (ret != LTTNG_OK) {
 586                 goto error;
 587         }
 588
 589         fd = kernctl_create_event(channel->fd, event->event);
 590         if (fd < 0) {
 591                 switch (-fd) {
 592                 case EEXIST:
 593                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 594                         break;
 595                 case ENOSYS:
 596                         WARN("Event type not implemented");
 597                         ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
 598                         break;
 599                 case ENOENT:
 600                         WARN("Event %s not found!", ev->name);
 601                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 602                         break;
 603                 default:
 604                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 605                         PERROR("create event ioctl");
 606                 }
 607                 goto free_event;
 608         }
 609
 610         event->type = ev->type;
 611         event->fd = fd;
 612         /* Prevent fd duplication after execlp() */
 613         err = fcntl(event->fd, F_SETFD, FD_CLOEXEC);
 614         if (err < 0) {
 615                 PERROR("fcntl session fd");
 616         }
 617
 618         if (filter) {
 619                 err = kernctl_filter(event->fd, filter);
 620                 if (err < 0) {
 621                         switch (-err) {
 622                         case ENOMEM:
 623                                 ret = LTTNG_ERR_FILTER_NOMEM;
 624                                 break;
 625                         default:
 626                                 ret = LTTNG_ERR_FILTER_INVAL;
 627                                 break;
 628                         }
 629                         goto filter_error;
 630                 }
 631         }
 632
 633         if (ev->type == LTTNG_EVENT_USERSPACE_PROBE) {
 634                 ret = userspace_probe_event_add_callsites(ev, channel->session,
 635                                 event->fd);
 636                 if (ret) {
 637                         goto add_callsite_error;
 638                 }
 639         }
 640
 641         err = kernctl_enable(event->fd);
 642         if (err < 0) {
 643                 switch (-err) {
 644                 case EEXIST:
 645                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 646                         break;
 647                 default:
 648                         PERROR("enable kernel event");
 649                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 650                         break;
 651                 }
 652                 goto enable_error;
 653         }
 654
 655         /* Add event to event list */
 656         cds_list_add(&event->list, &channel->events_list.head);
 657         channel->event_count++;
 658
 659         DBG("Event %s created (fd: %d)", ev->name, event->fd);
 660
 661         return 0;
 662
 663 add_callsite_error:
 664 enable_error:
 665 filter_error:
 666         {
 667                 int closeret;
 668
 669                 closeret = close(event->fd);
 670                 if (closeret) {
 671                         PERROR("close event fd");
 672                 }
 673         }
 674 free_event:
 675         free(event);
 676 error:
 677         return ret;
 678 }
 679
 680 /*
 681  * Disable a kernel channel.
 682  */
 683 int kernel_disable_channel(struct ltt_kernel_channel *chan)
 684 {
 685         int ret;
 686
 687         assert(chan);
 688
 689         ret = kernctl_disable(chan->fd);
 690         if (ret < 0) {
 691                 PERROR("disable chan ioctl");
 692                 goto error;
 693         }
 694
 695         chan->enabled = 0;
 696         DBG("Kernel channel %s disabled (fd: %d, key: %" PRIu64 ")",
 697                         chan->channel->name, chan->fd, chan->key);
 698
 699         return 0;
 700
 701 error:
 702         return ret;
 703 }
 704
 705 /*
 706  * Enable a kernel channel.
 707  */
 708 int kernel_enable_channel(struct ltt_kernel_channel *chan)
 709 {
 710         int ret;
 711
 712         assert(chan);
 713
 714         ret = kernctl_enable(chan->fd);
 715         if (ret < 0 && ret != -EEXIST) {
 716                 PERROR("Enable kernel chan");
 717                 goto error;
 718         }
 719
 720         chan->enabled = 1;
 721         DBG("Kernel channel %s enabled (fd: %d, key: %" PRIu64 ")",
 722                         chan->channel->name, chan->fd, chan->key);
 723
 724         return 0;
 725
 726 error:
 727         return ret;
 728 }
 729
 730 /*
 731  * Enable a kernel event.
 732  */
 733 int kernel_enable_event(struct ltt_kernel_event *event)
 734 {
 735         int ret;
 736
 737         assert(event);
 738
 739         ret = kernctl_enable(event->fd);
 740         if (ret < 0) {
 741                 switch (-ret) {
 742                 case EEXIST:
 743                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 744                         break;
 745                 default:
 746                         PERROR("enable kernel event");
 747                         break;
 748                 }
 749                 goto error;
 750         }
 751
 752         event->enabled = 1;
 753         DBG("Kernel event %s enabled (fd: %d)", event->event->name, event->fd);
 754
 755         return 0;
 756
 757 error:
 758         return ret;
 759 }
 760
 761 /*
 762  * Disable a kernel event.
 763  */
 764 int kernel_disable_event(struct ltt_kernel_event *event)
 765 {
 766         int ret;
 767
 768         assert(event);
 769
 770         ret = kernctl_disable(event->fd);
 771         if (ret < 0) {
 772                 switch (-ret) {
 773                 case EEXIST:
 774                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 775                         break;
 776                 default:
 777                         PERROR("disable kernel event");
 778                         break;
 779                 }
 780                 goto error;
 781         }
 782
 783         event->enabled = 0;
 784         DBG("Kernel event %s disabled (fd: %d)", event->event->name, event->fd);
 785
 786         return 0;
 787
 788 error:
 789         return ret;
 790 }
 791
 792 /*
 793  * Disable a kernel event notifier.
 794  */
 795 static
 796 int kernel_disable_event_notifier_rule(struct ltt_kernel_event_notifier_rule *event)
 797 {
 798         int ret;
 799
 800         assert(event);
 801
 802         rcu_read_lock();
 803         cds_lfht_del(kernel_token_to_event_notifier_rule_ht, &event->ht_node);
 804         rcu_read_unlock();
 805
 806         ret = kernctl_disable(event->fd);
 807         if (ret < 0) {
 808                 switch (-ret) {
 809                 case EEXIST:
 810                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 811                         break;
 812                 default:
 813                         PERROR("Failed to disable kernel event notifier: fd = %d, token = %" PRIu64,
 814                                         event->fd, event->token);
 815                         break;
 816                 }
 817                 goto error;
 818         }
 819
 820         event->enabled = 0;
 821         DBG("Disabled kernel event notifier: fd = %d, token = %" PRIu64,
 822                         event->fd, event->token);
 823
 824 error:
 825         return ret;
 826 }
 827
 828 static
 829 struct process_attr_tracker *_kernel_get_process_attr_tracker(
 830                 struct ltt_kernel_session *session,
 831                 enum lttng_process_attr process_attr)
 832 {
 833         switch (process_attr) {
 834         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 835                 return session->tracker_pid;
 836         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 837                 return session->tracker_vpid;
 838         case LTTNG_PROCESS_ATTR_USER_ID:
 839                 return session->tracker_uid;
 840         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 841                 return session->tracker_vuid;
 842         case LTTNG_PROCESS_ATTR_GROUP_ID:
 843                 return session->tracker_gid;
 844         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 845                 return session->tracker_vgid;
 846         default:
 847                 return NULL;
 848         }
 849 }
 850
 851 const struct process_attr_tracker *kernel_get_process_attr_tracker(
 852                 struct ltt_kernel_session *session,
 853                 enum lttng_process_attr process_attr)
 854 {
 855         return (const struct process_attr_tracker *)
 856                         _kernel_get_process_attr_tracker(session, process_attr);
 857 }
 858
 859 enum lttng_error_code kernel_process_attr_tracker_set_tracking_policy(
 860                 struct ltt_kernel_session *session,
 861                 enum lttng_process_attr process_attr,
 862                 enum lttng_tracking_policy policy)
 863 {
 864         int ret;
 865         enum lttng_error_code ret_code = LTTNG_OK;
 866         struct process_attr_tracker *tracker =
 867                         _kernel_get_process_attr_tracker(session, process_attr);
 868         enum lttng_tracking_policy previous_policy;
 869
 870         if (!tracker) {
 871                 ret_code = LTTNG_ERR_INVALID;
 872                 goto end;
 873         }
 874
 875         previous_policy = process_attr_tracker_get_tracking_policy(tracker);
 876         ret = process_attr_tracker_set_tracking_policy(tracker, policy);
 877         if (ret) {
 878                 ret_code = LTTNG_ERR_UNK;
 879                 goto end;
 880         }
 881
 882         if (previous_policy == policy) {
 883                 goto end;
 884         }
 885
 886         switch (policy) {
 887         case LTTNG_TRACKING_POLICY_INCLUDE_ALL:
 888                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 889                         /*
 890                          * Maintain a special case for the process ID process
 891                          * attribute tracker as it was the only supported
 892                          * attribute prior to 2.12.
 893                          */
 894                         ret = kernctl_track_pid(session->fd, -1);
 895                 } else {
 896                         ret = kernctl_track_id(session->fd, process_attr, -1);
 897                 }
 898                 break;
 899         case LTTNG_TRACKING_POLICY_EXCLUDE_ALL:
 900         case LTTNG_TRACKING_POLICY_INCLUDE_SET:
 901                 /* fall-through. */
 902                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 903                         /*
 904                          * Maintain a special case for the process ID process
 905                          * attribute tracker as it was the only supported
 906                          * attribute prior to 2.12.
 907                          */
 908                         ret = kernctl_untrack_pid(session->fd, -1);
 909                 } else {
 910                         ret = kernctl_untrack_id(session->fd, process_attr, -1);
 911                 }
 912                 break;
 913         default:
 914                 abort();
 915         }
 916         /* kern-ctl error handling */
 917         switch (-ret) {
 918         case 0:
 919                 ret_code = LTTNG_OK;
 920                 break;
 921         case EINVAL:
 922                 ret_code = LTTNG_ERR_INVALID;
 923                 break;
 924         case ENOMEM:
 925                 ret_code = LTTNG_ERR_NOMEM;
 926                 break;
 927         case EEXIST:
 928                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
 929                 break;
 930         default:
 931                 ret_code = LTTNG_ERR_UNK;
 932                 break;
 933         }
 934 end:
 935         return ret_code;
 936 }
 937
 938 enum lttng_error_code kernel_process_attr_tracker_inclusion_set_add_value(
 939                 struct ltt_kernel_session *session,
 940                 enum lttng_process_attr process_attr,
 941                 const struct process_attr_value *value)
 942 {
 943         int ret, integral_value;
 944         enum lttng_error_code ret_code;
 945         struct process_attr_tracker *tracker;
 946         enum process_attr_tracker_status status;
 947
 948         /*
 949          * Convert process attribute tracker value to the integral
 950          * representation required by the kern-ctl API.
 951          */
 952         switch (process_attr) {
 953         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 954         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 955                 integral_value = (int) value->value.pid;
 956                 break;
 957         case LTTNG_PROCESS_ATTR_USER_ID:
 958         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 959                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
 960                         uid_t uid;
 961
 962                         ret_code = utils_user_id_from_name(
 963                                         value->value.user_name, &uid);
 964                         if (ret_code != LTTNG_OK) {
 965                                 goto end;
 966                         }
 967                         integral_value = (int) uid;
 968                 } else {
 969                         integral_value = (int) value->value.uid;
 970                 }
 971                 break;
 972         case LTTNG_PROCESS_ATTR_GROUP_ID:
 973         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 974                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
 975                         gid_t gid;
 976
 977                         ret_code = utils_group_id_from_name(
 978                                         value->value.group_name, &gid);
 979                         if (ret_code != LTTNG_OK) {
 980                                 goto end;
 981                         }
 982                         integral_value = (int) gid;
 983                 } else {
 984                         integral_value = (int) value->value.gid;
 985                 }
 986                 break;
 987         default:
 988                 ret_code = LTTNG_ERR_INVALID;
 989                 goto end;
 990         }
 991
 992         tracker = _kernel_get_process_attr_tracker(session, process_attr);
 993         if (!tracker) {
 994                 ret_code = LTTNG_ERR_INVALID;
 995                 goto end;
 996         }
 997
 998         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
 999         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1000                 switch (status) {
1001                 case PROCESS_ATTR_TRACKER_STATUS_EXISTS:
1002                         ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1003                         break;
1004                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1005                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1006                         break;
1007                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1008                 default:
1009                         ret_code = LTTNG_ERR_UNK;
1010                         break;
1011                 }
1012                 goto end;
1013         }
1014
1015         DBG("Kernel track %s %d for session id %" PRIu64,
1016                         lttng_process_attr_to_string(process_attr),
1017                         integral_value, session->id);
1018         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1019                 /*
1020                  * Maintain a special case for the process ID process attribute
1021                  * tracker as it was the only supported attribute prior to 2.12.
1022                  */
1023                 ret = kernctl_track_pid(session->fd, integral_value);
1024         } else {
1025                 ret = kernctl_track_id(
1026                                 session->fd, process_attr, integral_value);
1027         }
1028         if (ret == 0) {
1029                 ret_code = LTTNG_OK;
1030                 goto end;
1031         }
1032
1033         kernel_wait_quiescent();
1034
1035         /* kern-ctl error handling */
1036         switch (-ret) {
1037         case 0:
1038                 ret_code = LTTNG_OK;
1039                 break;
1040         case EINVAL:
1041                 ret_code = LTTNG_ERR_INVALID;
1042                 break;
1043         case ENOMEM:
1044                 ret_code = LTTNG_ERR_NOMEM;
1045                 break;
1046         case EEXIST:
1047                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1048                 break;
1049         default:
1050                 ret_code = LTTNG_ERR_UNK;
1051                 break;
1052         }
1053
1054         /* Attempt to remove the value from the tracker. */
1055         status = process_attr_tracker_inclusion_set_remove_value(
1056                         tracker, value);
1057         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1058                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1059                                 lttng_process_attr_to_string(process_attr),
1060                                 integral_value);
1061         }
1062 end:
1063         return ret_code;
1064 }
1065
1066 enum lttng_error_code kernel_process_attr_tracker_inclusion_set_remove_value(
1067                 struct ltt_kernel_session *session,
1068                 enum lttng_process_attr process_attr,
1069                 const struct process_attr_value *value)
1070 {
1071         int ret, integral_value;
1072         enum lttng_error_code ret_code;
1073         struct process_attr_tracker *tracker;
1074         enum process_attr_tracker_status status;
1075
1076         /*
1077          * Convert process attribute tracker value to the integral
1078          * representation required by the kern-ctl API.
1079          */
1080         switch (process_attr) {
1081         case LTTNG_PROCESS_ATTR_PROCESS_ID:
1082         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
1083                 integral_value = (int) value->value.pid;
1084                 break;
1085         case LTTNG_PROCESS_ATTR_USER_ID:
1086         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
1087                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
1088                         uid_t uid;
1089
1090                         ret_code = utils_user_id_from_name(
1091                                         value->value.user_name, &uid);
1092                         if (ret_code != LTTNG_OK) {
1093                                 goto end;
1094                         }
1095                         integral_value = (int) uid;
1096                 } else {
1097                         integral_value = (int) value->value.uid;
1098                 }
1099                 break;
1100         case LTTNG_PROCESS_ATTR_GROUP_ID:
1101         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
1102                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
1103                         gid_t gid;
1104
1105                         ret_code = utils_group_id_from_name(
1106                                         value->value.group_name, &gid);
1107                         if (ret_code != LTTNG_OK) {
1108                                 goto end;
1109                         }
1110                         integral_value = (int) gid;
1111                 } else {
1112                         integral_value = (int) value->value.gid;
1113                 }
1114                 break;
1115         default:
1116                 ret_code = LTTNG_ERR_INVALID;
1117                 goto end;
1118         }
1119
1120         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1121         if (!tracker) {
1122                 ret_code = LTTNG_ERR_INVALID;
1123                 goto end;
1124         }
1125
1126         status = process_attr_tracker_inclusion_set_remove_value(
1127                         tracker, value);
1128         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1129                 switch (status) {
1130                 case PROCESS_ATTR_TRACKER_STATUS_MISSING:
1131                         ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1132                         break;
1133                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1134                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1135                         break;
1136                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1137                 default:
1138                         ret_code = LTTNG_ERR_UNK;
1139                         break;
1140                 }
1141                 goto end;
1142         }
1143
1144         DBG("Kernel track %s %d for session id %" PRIu64,
1145                         lttng_process_attr_to_string(process_attr),
1146                         integral_value, session->id);
1147         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1148                 /*
1149                  * Maintain a special case for the process ID process attribute
1150                  * tracker as it was the only supported attribute prior to 2.12.
1151                  */
1152                 ret = kernctl_untrack_pid(session->fd, integral_value);
1153         } else {
1154                 ret = kernctl_untrack_id(
1155                                 session->fd, process_attr, integral_value);
1156         }
1157         if (ret == 0) {
1158                 ret_code = LTTNG_OK;
1159                 goto end;
1160         }
1161         kernel_wait_quiescent();
1162
1163         /* kern-ctl error handling */
1164         switch (-ret) {
1165         case 0:
1166                 ret_code = LTTNG_OK;
1167                 break;
1168         case EINVAL:
1169                 ret_code = LTTNG_ERR_INVALID;
1170                 break;
1171         case ENOMEM:
1172                 ret_code = LTTNG_ERR_NOMEM;
1173                 break;
1174         case ENOENT:
1175                 ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1176                 break;
1177         default:
1178                 ret_code = LTTNG_ERR_UNK;
1179                 break;
1180         }
1181
1182         /* Attempt to add the value to the tracker. */
1183         status = process_attr_tracker_inclusion_set_add_value(
1184                         tracker, value);
1185         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1186                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1187                                 lttng_process_attr_to_string(process_attr),
1188                                 integral_value);
1189         }
1190 end:
1191         return ret_code;
1192 }
1193
1194 /*
1195  * Create kernel metadata, open from the kernel tracer and add it to the
1196  * kernel session.
1197  */
1198 int kernel_open_metadata(struct ltt_kernel_session *session)
1199 {
1200         int ret;
1201         struct ltt_kernel_metadata *lkm = NULL;
1202
1203         assert(session);
1204
1205         /* Allocate kernel metadata */
1206         lkm = trace_kernel_create_metadata();
1207         if (lkm == NULL) {
1208                 goto error;
1209         }
1210
1211         /* Kernel tracer metadata creation */
1212         ret = kernctl_open_metadata(session->fd, &lkm->conf->attr);
1213         if (ret < 0) {
1214                 goto error_open;
1215         }
1216
1217         lkm->fd = ret;
1218         lkm->key = ++next_kernel_channel_key;
1219         /* Prevent fd duplication after execlp() */
1220         ret = fcntl(lkm->fd, F_SETFD, FD_CLOEXEC);
1221         if (ret < 0) {
1222                 PERROR("fcntl session fd");
1223         }
1224
1225         session->metadata = lkm;
1226
1227         DBG("Kernel metadata opened (fd: %d)", lkm->fd);
1228
1229         return 0;
1230
1231 error_open:
1232         trace_kernel_destroy_metadata(lkm);
1233 error:
1234         return -1;
1235 }
1236
1237 /*
1238  * Start tracing session.
1239  */
1240 int kernel_start_session(struct ltt_kernel_session *session)
1241 {
1242         int ret;
1243
1244         assert(session);
1245
1246         ret = kernctl_start_session(session->fd);
1247         if (ret < 0) {
1248                 PERROR("ioctl start session");
1249                 goto error;
1250         }
1251
1252         DBG("Kernel session started");
1253
1254         return 0;
1255
1256 error:
1257         return ret;
1258 }
1259
1260 /*
1261  * Make a kernel wait to make sure in-flight probe have completed.
1262  */
1263 void kernel_wait_quiescent(void)
1264 {
1265         int ret;
1266         int fd = kernel_tracer_fd;
1267
1268         DBG("Kernel quiescent wait on %d", fd);
1269
1270         ret = kernctl_wait_quiescent(fd);
1271         if (ret < 0) {
1272                 PERROR("wait quiescent ioctl");
1273                 ERR("Kernel quiescent wait failed");
1274         }
1275 }
1276
1277 /*
1278  *  Force flush buffer of metadata.
1279  */
1280 int kernel_metadata_flush_buffer(int fd)
1281 {
1282         int ret;
1283
1284         DBG("Kernel flushing metadata buffer on fd %d", fd);
1285
1286         ret = kernctl_buffer_flush(fd);
1287         if (ret < 0) {
1288                 ERR("Fail to flush metadata buffers %d (ret: %d)", fd, ret);
1289         }
1290
1291         return 0;
1292 }
1293
1294 /*
1295  * Force flush buffer for channel.
1296  */
1297 int kernel_flush_buffer(struct ltt_kernel_channel *channel)
1298 {
1299         int ret;
1300         struct ltt_kernel_stream *stream;
1301
1302         assert(channel);
1303
1304         DBG("Flush buffer for channel %s", channel->channel->name);
1305
1306         cds_list_for_each_entry(stream, &channel->stream_list.head, list) {
1307                 DBG("Flushing channel stream %d", stream->fd);
1308                 ret = kernctl_buffer_flush(stream->fd);
1309                 if (ret < 0) {
1310                         PERROR("ioctl");
1311                         ERR("Fail to flush buffer for stream %d (ret: %d)",
1312                                         stream->fd, ret);
1313                 }
1314         }
1315
1316         return 0;
1317 }
1318
1319 /*
1320  * Stop tracing session.
1321  */
1322 int kernel_stop_session(struct ltt_kernel_session *session)
1323 {
1324         int ret;
1325
1326         assert(session);
1327
1328         ret = kernctl_stop_session(session->fd);
1329         if (ret < 0) {
1330                 goto error;
1331         }
1332
1333         DBG("Kernel session stopped");
1334
1335         return 0;
1336
1337 error:
1338         return ret;
1339 }
1340
1341 /*
1342  * Open stream of channel, register it to the kernel tracer and add it
1343  * to the stream list of the channel.
1344  *
1345  * Note: given that the streams may appear in random order wrt CPU
1346  * number (e.g. cpu hotplug), the index value of the stream number in
1347  * the stream name is not necessarily linked to the CPU number.
1348  *
1349  * Return the number of created stream. Else, a negative value.
1350  */
1351 int kernel_open_channel_stream(struct ltt_kernel_channel *channel)
1352 {
1353         int ret;
1354         struct ltt_kernel_stream *lks;
1355
1356         assert(channel);
1357
1358         while ((ret = kernctl_create_stream(channel->fd)) >= 0) {
1359                 lks = trace_kernel_create_stream(channel->channel->name,
1360                                 channel->stream_count);
1361                 if (lks == NULL) {
1362                         ret = close(ret);
1363                         if (ret) {
1364                                 PERROR("close");
1365                         }
1366                         goto error;
1367                 }
1368
1369                 lks->fd = ret;
1370                 /* Prevent fd duplication after execlp() */
1371                 ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
1372                 if (ret < 0) {
1373                         PERROR("fcntl session fd");
1374                 }
1375
1376                 lks->tracefile_size = channel->channel->attr.tracefile_size;
1377                 lks->tracefile_count = channel->channel->attr.tracefile_count;
1378
1379                 /* Add stream to channel stream list */
1380                 cds_list_add(&lks->list, &channel->stream_list.head);
1381                 channel->stream_count++;
1382
1383                 DBG("Kernel stream %s created (fd: %d, state: %d)", lks->name, lks->fd,
1384                                 lks->state);
1385         }
1386
1387         return channel->stream_count;
1388
1389 error:
1390         return -1;
1391 }
1392
1393 /*
1394  * Open the metadata stream and set it to the kernel session.
1395  */
1396 int kernel_open_metadata_stream(struct ltt_kernel_session *session)
1397 {
1398         int ret;
1399
1400         assert(session);
1401
1402         ret = kernctl_create_stream(session->metadata->fd);
1403         if (ret < 0) {
1404                 PERROR("kernel create metadata stream");
1405                 goto error;
1406         }
1407
1408         DBG("Kernel metadata stream created (fd: %d)", ret);
1409         session->metadata_stream_fd = ret;
1410         /* Prevent fd duplication after execlp() */
1411         ret = fcntl(session->metadata_stream_fd, F_SETFD, FD_CLOEXEC);
1412         if (ret < 0) {
1413                 PERROR("fcntl session fd");
1414         }
1415
1416         return 0;
1417
1418 error:
1419         return -1;
1420 }
1421
1422 /*
1423  * Get the event list from the kernel tracer and return the number of elements.
1424  */
1425 ssize_t kernel_list_events(struct lttng_event **events)
1426 {
1427         int fd, ret;
1428         char *event;
1429         size_t nbmem, count = 0;
1430         FILE *fp;
1431         struct lttng_event *elist;
1432
1433         assert(events);
1434
1435         fd = kernctl_tracepoint_list(kernel_tracer_fd);
1436         if (fd < 0) {
1437                 PERROR("kernel tracepoint list");
1438                 goto error;
1439         }
1440
1441         fp = fdopen(fd, "r");
1442         if (fp == NULL) {
1443                 PERROR("kernel tracepoint list fdopen");
1444                 goto error_fp;
1445         }
1446
1447         /*
1448          * Init memory size counter
1449          * See kernel-ctl.h for explanation of this value
1450          */
1451         nbmem = KERNEL_EVENT_INIT_LIST_SIZE;
1452         elist = zmalloc(sizeof(struct lttng_event) * nbmem);
1453         if (elist == NULL) {
1454                 PERROR("alloc list events");
1455                 count = -ENOMEM;
1456                 goto end;
1457         }
1458
1459         while (fscanf(fp, "event { name = %m[^;]; };\n", &event) == 1) {
1460                 if (count >= nbmem) {
1461                         struct lttng_event *new_elist;
1462                         size_t new_nbmem;
1463
1464                         new_nbmem = nbmem << 1;
1465                         DBG("Reallocating event list from %zu to %zu bytes",
1466                                         nbmem, new_nbmem);
1467                         new_elist = realloc(elist, new_nbmem * sizeof(struct lttng_event));
1468                         if (new_elist == NULL) {
1469                                 PERROR("realloc list events");
1470                                 free(event);
1471                                 free(elist);
1472                                 count = -ENOMEM;
1473                                 goto end;
1474                         }
1475                         /* Zero the new memory */
1476                         memset(new_elist + nbmem, 0,
1477                                 (new_nbmem - nbmem) * sizeof(struct lttng_event));
1478                         nbmem = new_nbmem;
1479                         elist = new_elist;
1480                 }
1481                 strncpy(elist[count].name, event, LTTNG_SYMBOL_NAME_LEN);
1482                 elist[count].name[LTTNG_SYMBOL_NAME_LEN - 1] = '\0';
1483                 elist[count].enabled = -1;
1484                 count++;
1485                 free(event);
1486         }
1487
1488         *events = elist;
1489         DBG("Kernel list events done (%zu events)", count);
1490 end:
1491         ret = fclose(fp);       /* closes both fp and fd */
1492         if (ret) {
1493                 PERROR("fclose");
1494         }
1495         return count;
1496
1497 error_fp:
1498         ret = close(fd);
1499         if (ret) {
1500                 PERROR("close");
1501         }
1502 error:
1503         return -1;
1504 }
1505
1506 /*
1507  * Get kernel version and validate it.
1508  */
1509 int kernel_validate_version(struct lttng_kernel_tracer_version *version,
1510                 struct lttng_kernel_tracer_abi_version *abi_version)
1511 {
1512         int ret;
1513
1514         ret = kernctl_tracer_version(kernel_tracer_fd, version);
1515         if (ret < 0) {
1516                 ERR("Failed to retrieve the lttng-modules version");
1517                 goto error;
1518         }
1519
1520         /* Validate version */
1521         if (version->major != VERSION_MAJOR) {
1522                 ERR("Kernel tracer major version (%d) is not compatible with lttng-tools major version (%d)",
1523                         version->major, VERSION_MAJOR);
1524                 goto error_version;
1525         }
1526         ret = kernctl_tracer_abi_version(kernel_tracer_fd, abi_version);
1527         if (ret < 0) {
1528                 ERR("Failed to retrieve lttng-modules ABI version");
1529                 goto error;
1530         }
1531         if (abi_version->major != LTTNG_MODULES_ABI_MAJOR_VERSION) {
1532                 ERR("Kernel tracer ABI version (%d.%d) does not match the expected ABI major version (%d.*)",
1533                         abi_version->major, abi_version->minor,
1534                         LTTNG_MODULES_ABI_MAJOR_VERSION);
1535                 goto error;
1536         }
1537         DBG2("Kernel tracer version validated (%d.%d, ABI %d.%d)",
1538                         version->major, version->minor,
1539                         abi_version->major, abi_version->minor);
1540         return 0;
1541
1542 error_version:
1543         ret = -1;
1544
1545 error:
1546         ERR("Kernel tracer version check failed; kernel tracing will not be available");
1547         return ret;
1548 }
1549
1550 /*
1551  * Kernel work-arounds called at the start of sessiond main().
1552  */
1553 int init_kernel_workarounds(void)
1554 {
1555         int ret;
1556         FILE *fp;
1557
1558         /*
1559          * boot_id needs to be read once before being used concurrently
1560          * to deal with a Linux kernel race. A fix is proposed for
1561          * upstream, but the work-around is needed for older kernels.
1562          */
1563         fp = fopen("/proc/sys/kernel/random/boot_id", "r");
1564         if (!fp) {
1565                 goto end_boot_id;
1566         }
1567         while (!feof(fp)) {
1568                 char buf[37] = "";
1569
1570                 ret = fread(buf, 1, sizeof(buf), fp);
1571                 if (ret < 0) {
1572                         /* Ignore error, we don't really care */
1573                 }
1574         }
1575         ret = fclose(fp);
1576         if (ret) {
1577                 PERROR("fclose");
1578         }
1579 end_boot_id:
1580         return 0;
1581 }
1582
1583 /*
1584  * Teardown of a kernel session, keeping data required by destroy notifiers.
1585  */
1586 void kernel_destroy_session(struct ltt_kernel_session *ksess)
1587 {
1588         struct lttng_trace_chunk *trace_chunk;
1589
1590         if (ksess == NULL) {
1591                 DBG3("No kernel session when tearing down session");
1592                 return;
1593         }
1594
1595         DBG("Tearing down kernel session");
1596         trace_chunk = ksess->current_trace_chunk;
1597
1598         /*
1599          * Destroy channels on the consumer if at least one FD has been sent and we
1600          * are in no output mode because the streams are in *no* monitor mode so we
1601          * have to send a command to clean them up or else they leaked.
1602          */
1603         if (!ksess->output_traces && ksess->consumer_fds_sent) {
1604                 int ret;
1605                 struct consumer_socket *socket;
1606                 struct lttng_ht_iter iter;
1607
1608                 /* For each consumer socket. */
1609                 rcu_read_lock();
1610                 cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
1611                                 socket, node.node) {
1612                         struct ltt_kernel_channel *chan;
1613
1614                         /* For each channel, ask the consumer to destroy it. */
1615                         cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1616                                 ret = kernel_consumer_destroy_channel(socket, chan);
1617                                 if (ret < 0) {
1618                                         /* Consumer is probably dead. Use next socket. */
1619                                         continue;
1620                                 }
1621                         }
1622                 }
1623                 rcu_read_unlock();
1624         }
1625
1626         /* Close any relayd session */
1627         consumer_output_send_destroy_relayd(ksess->consumer);
1628
1629         trace_kernel_destroy_session(ksess);
1630         lttng_trace_chunk_put(trace_chunk);
1631 }
1632
1633 /* Teardown of data required by destroy notifiers. */
1634 void kernel_free_session(struct ltt_kernel_session *ksess)
1635 {
1636         if (ksess == NULL) {
1637                 return;
1638         }
1639         trace_kernel_free_session(ksess);
1640 }
1641
1642 /*
1643  * Destroy a kernel channel object. It does not do anything on the tracer side.
1644  */
1645 void kernel_destroy_channel(struct ltt_kernel_channel *kchan)
1646 {
1647         struct ltt_kernel_session *ksess = NULL;
1648
1649         assert(kchan);
1650         assert(kchan->channel);
1651
1652         DBG3("Kernel destroy channel %s", kchan->channel->name);
1653
1654         /* Update channel count of associated session. */
1655         if (kchan->session) {
1656                 /* Keep pointer reference so we can update it after the destroy. */
1657                 ksess = kchan->session;
1658         }
1659
1660         trace_kernel_destroy_channel(kchan);
1661
1662         /*
1663          * At this point the kernel channel is not visible anymore. This is safe
1664          * since in order to work on a visible kernel session, the tracing session
1665          * lock (ltt_session.lock) MUST be acquired.
1666          */
1667         if (ksess) {
1668                 ksess->channel_count--;
1669         }
1670 }
1671
1672 /*
1673  * Take a snapshot for a given kernel session.
1674  *
1675  * Return LTTNG_OK on success or else return a LTTNG_ERR code.
1676  */
1677 enum lttng_error_code kernel_snapshot_record(
1678                 struct ltt_kernel_session *ksess,
1679                 const struct consumer_output *output, int wait,
1680                 uint64_t nb_packets_per_stream)
1681 {
1682         int err, ret, saved_metadata_fd;
1683         enum lttng_error_code status = LTTNG_OK;
1684         struct consumer_socket *socket;
1685         struct lttng_ht_iter iter;
1686         struct ltt_kernel_metadata *saved_metadata;
1687         char *trace_path = NULL;
1688         size_t consumer_path_offset = 0;
1689
1690         assert(ksess);
1691         assert(ksess->consumer);
1692         assert(output);
1693
1694         DBG("Kernel snapshot record started");
1695
1696         /* Save current metadata since the following calls will change it. */
1697         saved_metadata = ksess->metadata;
1698         saved_metadata_fd = ksess->metadata_stream_fd;
1699
1700         rcu_read_lock();
1701
1702         ret = kernel_open_metadata(ksess);
1703         if (ret < 0) {
1704                 status = LTTNG_ERR_KERN_META_FAIL;
1705                 goto error;
1706         }
1707
1708         ret = kernel_open_metadata_stream(ksess);
1709         if (ret < 0) {
1710                 status = LTTNG_ERR_KERN_META_FAIL;
1711                 goto error_open_stream;
1712         }
1713
1714         trace_path = setup_channel_trace_path(ksess->consumer,
1715                         DEFAULT_KERNEL_TRACE_DIR, &consumer_path_offset);
1716         if (!trace_path) {
1717                 status = LTTNG_ERR_INVALID;
1718                 goto error;
1719         }
1720         /* Send metadata to consumer and snapshot everything. */
1721         cds_lfht_for_each_entry(output->socks->ht, &iter.iter,
1722                         socket, node.node) {
1723                 struct ltt_kernel_channel *chan;
1724
1725                 pthread_mutex_lock(socket->lock);
1726                 /* This stream must not be monitored by the consumer. */
1727                 ret = kernel_consumer_add_metadata(socket, ksess, 0);
1728                 pthread_mutex_unlock(socket->lock);
1729                 if (ret < 0) {
1730                         status = LTTNG_ERR_KERN_META_FAIL;
1731                         goto error_consumer;
1732                 }
1733
1734                 /* For each channel, ask the consumer to snapshot it. */
1735                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1736                         status = consumer_snapshot_channel(socket, chan->key, output, 0,
1737                                         ksess->uid, ksess->gid,
1738                                         &trace_path[consumer_path_offset], wait,
1739                                         nb_packets_per_stream);
1740                         if (status != LTTNG_OK) {
1741                                 (void) kernel_consumer_destroy_metadata(socket,
1742                                                 ksess->metadata);
1743                                 goto error_consumer;
1744                         }
1745                 }
1746
1747                 /* Snapshot metadata, */
1748                 status = consumer_snapshot_channel(socket, ksess->metadata->key, output,
1749                                 1, ksess->uid, ksess->gid, &trace_path[consumer_path_offset],
1750                                 wait, 0);
1751                 if (status != LTTNG_OK) {
1752                         goto error_consumer;
1753                 }
1754
1755                 /*
1756                  * The metadata snapshot is done, ask the consumer to destroy it since
1757                  * it's not monitored on the consumer side.
1758                  */
1759                 (void) kernel_consumer_destroy_metadata(socket, ksess->metadata);
1760         }
1761
1762 error_consumer:
1763         /* Close newly opened metadata stream. It's now on the consumer side. */
1764         err = close(ksess->metadata_stream_fd);
1765         if (err < 0) {
1766                 PERROR("close snapshot kernel");
1767         }
1768
1769 error_open_stream:
1770         trace_kernel_destroy_metadata(ksess->metadata);
1771 error:
1772         /* Restore metadata state.*/
1773         ksess->metadata = saved_metadata;
1774         ksess->metadata_stream_fd = saved_metadata_fd;
1775         rcu_read_unlock();
1776         free(trace_path);
1777         return status;
1778 }
1779
1780 /*
1781  * Get the syscall mask array from the kernel tracer.
1782  *
1783  * Return 0 on success else a negative value. In both case, syscall_mask should
1784  * be freed.
1785  */
1786 int kernel_syscall_mask(int chan_fd, char **syscall_mask, uint32_t *nr_bits)
1787 {
1788         assert(syscall_mask);
1789         assert(nr_bits);
1790
1791         return kernctl_syscall_mask(chan_fd, syscall_mask, nr_bits);
1792 }
1793
1794 static
1795 int kernel_tracer_abi_greater_or_equal(unsigned int major, unsigned int minor)
1796 {
1797         int ret;
1798         struct lttng_kernel_tracer_abi_version abi;
1799
1800         ret = kernctl_tracer_abi_version(kernel_tracer_fd, &abi);
1801         if (ret < 0) {
1802                 ERR("Failed to retrieve lttng-modules ABI version");
1803                 goto error;
1804         }
1805
1806         ret = abi.major > major || (abi.major == major && abi.minor >= minor);
1807 error:
1808         return ret;
1809 }
1810
1811 /*
1812  * Check for the support of the RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS via abi
1813  * version number.
1814  *
1815  * Return 1 on success, 0 when feature is not supported, negative value in case
1816  * of errors.
1817  */
1818 int kernel_supports_ring_buffer_snapshot_sample_positions(void)
1819 {
1820         /*
1821          * RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS was introduced in 2.3
1822          */
1823         return kernel_tracer_abi_greater_or_equal(2, 3);
1824 }
1825
1826 /*
1827  * Check for the support of the packet sequence number via abi version number.
1828  *
1829  * Return 1 on success, 0 when feature is not supported, negative value in case
1830  * of errors.
1831  */
1832 int kernel_supports_ring_buffer_packet_sequence_number(void)
1833 {
1834         /*
1835          * Packet sequence number was introduced in LTTng 2.8,
1836          * lttng-modules ABI 2.1.
1837          */
1838         return kernel_tracer_abi_greater_or_equal(2, 1);
1839 }
1840
1841 /*
1842  * Check for the support of event notifiers via abi version number.
1843  *
1844  * Return 1 on success, 0 when feature is not supported, negative value in case
1845  * of errors.
1846  */
1847 int kernel_supports_event_notifiers(void)
1848 {
1849         /*
1850          * Event notifiers were introduced in LTTng 2.13, lttng-modules ABI 2.6.
1851          */
1852         return kernel_tracer_abi_greater_or_equal(2, 6);
1853 }
1854
1855 /*
1856  * Rotate a kernel session.
1857  *
1858  * Return LTTNG_OK on success or else an LTTng error code.
1859  */
1860 enum lttng_error_code kernel_rotate_session(struct ltt_session *session)
1861 {
1862         int ret;
1863         enum lttng_error_code status = LTTNG_OK;
1864         struct consumer_socket *socket;
1865         struct lttng_ht_iter iter;
1866         struct ltt_kernel_session *ksess = session->kernel_session;
1867
1868         assert(ksess);
1869         assert(ksess->consumer);
1870
1871         DBG("Rotate kernel session %s started (session %" PRIu64 ")",
1872                         session->name, session->id);
1873
1874         rcu_read_lock();
1875
1876         /*
1877          * Note that this loop will end after one iteration given that there is
1878          * only one kernel consumer.
1879          */
1880         cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
1881                         socket, node.node) {
1882                 struct ltt_kernel_channel *chan;
1883
1884                 /* For each channel, ask the consumer to rotate it. */
1885                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1886                         DBG("Rotate kernel channel %" PRIu64 ", session %s",
1887                                         chan->key, session->name);
1888                         ret = consumer_rotate_channel(socket, chan->key,
1889                                         ksess->uid, ksess->gid, ksess->consumer,
1890                                         /* is_metadata_channel */ false);
1891                         if (ret < 0) {
1892                                 status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1893                                 goto error;
1894                         }
1895                 }
1896
1897                 /*
1898                  * Rotate the metadata channel.
1899                  */
1900                 ret = consumer_rotate_channel(socket, ksess->metadata->key,
1901                                 ksess->uid, ksess->gid, ksess->consumer,
1902                                 /* is_metadata_channel */ true);
1903                 if (ret < 0) {
1904                         status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1905                         goto error;
1906                 }
1907         }
1908
1909 error:
1910         rcu_read_unlock();
1911         return status;
1912 }
1913
1914 enum lttng_error_code kernel_create_channel_subdirectories(
1915                 const struct ltt_kernel_session *ksess)
1916 {
1917         enum lttng_error_code ret = LTTNG_OK;
1918         enum lttng_trace_chunk_status chunk_status;
1919
1920         rcu_read_lock();
1921         assert(ksess->current_trace_chunk);
1922
1923         /*
1924          * Create the index subdirectory which will take care
1925          * of implicitly creating the channel's path.
1926          */
1927         chunk_status = lttng_trace_chunk_create_subdirectory(
1928                         ksess->current_trace_chunk,
1929                         DEFAULT_KERNEL_TRACE_DIR "/" DEFAULT_INDEX_DIR);
1930         if (chunk_status != LTTNG_TRACE_CHUNK_STATUS_OK) {
1931                 ret = LTTNG_ERR_CREATE_DIR_FAIL;
1932                 goto error;
1933         }
1934 error:
1935         rcu_read_unlock();
1936         return ret;
1937 }
1938
1939 /*
1940  * Setup necessary data for kernel tracer action.
1941  */
1942 LTTNG_HIDDEN
1943 int init_kernel_tracer(void)
1944 {
1945         int ret;
1946         bool is_root = !getuid();
1947
1948         /* Modprobe lttng kernel modules */
1949         ret = modprobe_lttng_control();
1950         if (ret < 0) {
1951                 goto error;
1952         }
1953
1954         /* Open debugfs lttng */
1955         kernel_tracer_fd = open(module_proc_lttng, O_RDWR);
1956         if (kernel_tracer_fd < 0) {
1957                 DBG("Failed to open %s", module_proc_lttng);
1958                 goto error_open;
1959         }
1960
1961         /* Validate kernel version */
1962         ret = kernel_validate_version(&kernel_tracer_version,
1963                         &kernel_tracer_abi_version);
1964         if (ret < 0) {
1965                 goto error_version;
1966         }
1967
1968         ret = modprobe_lttng_data();
1969         if (ret < 0) {
1970                 goto error_modules;
1971         }
1972
1973         ret = kernel_supports_ring_buffer_snapshot_sample_positions();
1974         if (ret < 0) {
1975                 goto error_modules;
1976         }
1977         if (ret < 1) {
1978                 WARN("Kernel tracer does not support buffer monitoring. "
1979                         "The monitoring timer of channels in the kernel domain "
1980                         "will be set to 0 (disabled).");
1981         }
1982
1983         ret = kernel_supports_event_notifiers();
1984         if (ret < 0) {
1985                 ERR("Failed to check for kernel tracer event notifier support");
1986                 goto error_modules;
1987         }
1988         ret = kernel_create_event_notifier_group(&kernel_tracer_event_notifier_group_fd);
1989         if (ret < 0) {
1990                 /* This is not fatal. */
1991                 WARN("Failed to create kernel event notifier group");
1992                 kernel_tracer_event_notifier_group_fd = -1;
1993         } else {
1994                 enum event_notifier_error_accounting_status error_accounting_status;
1995                 enum lttng_error_code error_code_ret =
1996                                 kernel_create_event_notifier_group_notification_fd(
1997                                                 &kernel_tracer_event_notifier_group_notification_fd);
1998
1999                 if (error_code_ret != LTTNG_OK) {
2000                         goto error_modules;
2001                 }
2002
2003                 error_accounting_status = event_notifier_error_accounting_register_kernel(
2004                                 kernel_tracer_event_notifier_group_fd);
2005                 if (error_accounting_status != EVENT_NOTIFIER_ERROR_ACCOUNTING_STATUS_OK) {
2006                         ERR("Failed to initialize event notifier error accounting for kernel tracer");
2007                         error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_ERROR_ACCOUNTING;
2008                         goto error_modules;
2009                 }
2010
2011                 kernel_token_to_event_notifier_rule_ht = cds_lfht_new(
2012                                 DEFAULT_HT_SIZE, 1, 0,
2013                                 CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING,
2014                                 NULL);
2015                 if (!kernel_token_to_event_notifier_rule_ht) {
2016                         goto error_token_ht;
2017                 }
2018         }
2019
2020         DBG("Kernel tracer initialized: kernel tracer fd = %d, event notifier group fd = %d, event notifier group notification fd = %d",
2021                         kernel_tracer_fd, kernel_tracer_event_notifier_group_fd,
2022                         kernel_tracer_event_notifier_group_notification_fd);
2023
2024         ret = syscall_init_table(kernel_tracer_fd);
2025         if (ret < 0) {
2026                 ERR("Unable to populate syscall table. Syscall tracing won't "
2027                         "work for this session daemon.");
2028         }
2029
2030         return 0;
2031
2032 error_version:
2033         modprobe_remove_lttng_control();
2034         ret = close(kernel_tracer_fd);
2035         if (ret) {
2036                 PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2037                                 kernel_tracer_fd);
2038         }
2039         kernel_tracer_fd = -1;
2040         return LTTNG_ERR_KERN_VERSION;
2041
2042
2043 error_token_ht:
2044         ret = close(kernel_tracer_event_notifier_group_notification_fd);
2045         if (ret) {
2046                 PERROR("Failed to close kernel tracer event notifier group notification file descriptor: fd = %d",
2047                                 kernel_tracer_event_notifier_group_notification_fd);
2048         }
2049
2050 error_modules:
2051         ret = close(kernel_tracer_event_notifier_group_fd);
2052         if (ret) {
2053                 PERROR("Failed to close kernel tracer event notifier group file descriptor: fd = %d",
2054                                 kernel_tracer_event_notifier_group_fd);
2055         }
2056
2057         ret = close(kernel_tracer_fd);
2058         if (ret) {
2059                 PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2060                                 kernel_tracer_fd);
2061         }
2062
2063 error_open:
2064         modprobe_remove_lttng_control();
2065
2066 error:
2067         WARN("No kernel tracer available");
2068         kernel_tracer_fd = -1;
2069         if (!is_root) {
2070                 return LTTNG_ERR_NEED_ROOT_SESSIOND;
2071         } else {
2072                 return LTTNG_ERR_KERN_NA;
2073         }
2074 }
2075
2076 LTTNG_HIDDEN
2077 void cleanup_kernel_tracer(void)
2078 {
2079         DBG2("Closing kernel event notifier group notification file descriptor");
2080         if (kernel_tracer_event_notifier_group_notification_fd >= 0) {
2081                 int ret = notification_thread_command_remove_tracer_event_source(
2082                                 notification_thread_handle,
2083                                 kernel_tracer_event_notifier_group_notification_fd);
2084                 if (ret != LTTNG_OK) {
2085                         ERR("Failed to remove kernel event notifier notification from notification thread");
2086                 }
2087
2088                 ret = close(kernel_tracer_event_notifier_group_notification_fd);
2089                 if (ret) {
2090                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2091                                         kernel_tracer_event_notifier_group_notification_fd);
2092                 }
2093
2094                 kernel_tracer_event_notifier_group_notification_fd = -1;
2095         }
2096
2097         if (kernel_token_to_event_notifier_rule_ht) {
2098                 const int ret = cds_lfht_destroy(
2099                                 kernel_token_to_event_notifier_rule_ht, NULL);
2100                 assert(ret == 0);
2101         }
2102
2103         DBG2("Closing kernel event notifier group file descriptor");
2104         if (kernel_tracer_event_notifier_group_fd >= 0) {
2105                 const int ret = close(kernel_tracer_event_notifier_group_fd);
2106
2107                 if (ret) {
2108                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
2109                                         kernel_tracer_event_notifier_group_fd);
2110                 }
2111
2112                 kernel_tracer_event_notifier_group_fd = -1;
2113         }
2114
2115         DBG2("Closing kernel fd");
2116         if (kernel_tracer_fd >= 0) {
2117                 const int ret = close(kernel_tracer_fd);
2118
2119                 if (ret) {
2120                         PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2121                                         kernel_tracer_fd);
2122                 }
2123
2124                 kernel_tracer_fd = -1;
2125         }
2126
2127         DBG("Unloading kernel modules");
2128         modprobe_remove_lttng_all();
2129         free(syscall_table);
2130 }
2131
2132 LTTNG_HIDDEN
2133 bool kernel_tracer_is_initialized(void)
2134 {
2135         return kernel_tracer_fd >= 0;
2136 }
2137
2138 /*
2139  *  Clear a kernel session.
2140  *
2141  * Return LTTNG_OK on success or else an LTTng error code.
2142  */
2143 enum lttng_error_code kernel_clear_session(struct ltt_session *session)
2144 {
2145         int ret;
2146         enum lttng_error_code status = LTTNG_OK;
2147         struct consumer_socket *socket;
2148         struct lttng_ht_iter iter;
2149         struct ltt_kernel_session *ksess = session->kernel_session;
2150
2151         assert(ksess);
2152         assert(ksess->consumer);
2153
2154         DBG("Clear kernel session %s (session %" PRIu64 ")",
2155                         session->name, session->id);
2156
2157         rcu_read_lock();
2158
2159         if (ksess->active) {
2160                 ERR("Expecting inactive session %s (%" PRIu64 ")", session->name, session->id);
2161                 status = LTTNG_ERR_FATAL;
2162                 goto end;
2163         }
2164
2165         /*
2166          * Note that this loop will end after one iteration given that there is
2167          * only one kernel consumer.
2168          */
2169         cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
2170                         socket, node.node) {
2171                 struct ltt_kernel_channel *chan;
2172
2173                 /* For each channel, ask the consumer to clear it. */
2174                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
2175                         DBG("Clear kernel channel %" PRIu64 ", session %s",
2176                                         chan->key, session->name);
2177                         ret = consumer_clear_channel(socket, chan->key);
2178                         if (ret < 0) {
2179                                 goto error;
2180                         }
2181                 }
2182
2183                 if (!ksess->metadata) {
2184                         /*
2185                          * Nothing to do for the metadata.
2186                          * This is a snapshot session.
2187                          * The metadata is genererated on the fly.
2188                          */
2189                         continue;
2190                 }
2191
2192                 /*
2193                  * Clear the metadata channel.
2194                  * Metadata channel is not cleared per se but we still need to
2195                  * perform a rotation operation on it behind the scene.
2196                  */
2197                 ret = consumer_clear_channel(socket, ksess->metadata->key);
2198                 if (ret < 0) {
2199                         goto error;
2200                 }
2201         }
2202
2203         goto end;
2204 error:
2205         switch (-ret) {
2206         case LTTCOMM_CONSUMERD_RELAYD_CLEAR_DISALLOWED:
2207               status = LTTNG_ERR_CLEAR_RELAY_DISALLOWED;
2208               break;
2209         default:
2210               status = LTTNG_ERR_CLEAR_FAIL_CONSUMER;
2211               break;
2212         }
2213 end:
2214         rcu_read_unlock();
2215         return status;
2216 }
2217
2218 enum lttng_error_code kernel_create_event_notifier_group_notification_fd(
2219                 int *event_notifier_group_notification_fd)
2220 {
2221         int local_fd = -1, ret;
2222         enum lttng_error_code error_code_ret;
2223
2224         assert(event_notifier_group_notification_fd);
2225
2226         ret = kernctl_create_event_notifier_group_notification_fd(
2227                         kernel_tracer_event_notifier_group_fd);
2228         if (ret < 0) {
2229                 PERROR("Failed to create kernel event notifier group notification file descriptor");
2230                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2231                 goto error;
2232         }
2233
2234         local_fd = ret;
2235
2236         /* Prevent fd duplication after execlp(). */
2237         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
2238         if (ret < 0) {
2239                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group notification file descriptor: fd = %d",
2240                                 local_fd);
2241                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2242                 goto error;
2243         }
2244
2245         DBG("Created kernel notifier group notification file descriptor: fd = %d",
2246                         local_fd);
2247         error_code_ret = LTTNG_OK;
2248         *event_notifier_group_notification_fd = local_fd;
2249         local_fd = -1;
2250
2251 error:
2252         if (local_fd >= 0) {
2253                 ret = close(local_fd);
2254                 if (ret) {
2255                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2256                                         local_fd);
2257                 }
2258         }
2259
2260         return error_code_ret;
2261 }
2262
2263 enum lttng_error_code kernel_destroy_event_notifier_group_notification_fd(
2264                 int event_notifier_group_notification_fd)
2265 {
2266         enum lttng_error_code ret_code = LTTNG_OK;
2267
2268         DBG("Closing event notifier group notification file descriptor: fd = %d",
2269                         event_notifier_group_notification_fd);
2270         if (event_notifier_group_notification_fd >= 0) {
2271                 const int ret = close(event_notifier_group_notification_fd);
2272                 if (ret) {
2273                         PERROR("Failed to close event notifier group notification file descriptor: fd = %d",
2274                                         event_notifier_group_notification_fd);
2275                 }
2276         }
2277
2278         return ret_code;
2279 }
2280
2281 static
2282 unsigned long hash_trigger(const struct lttng_trigger *trigger)
2283 {
2284         const struct lttng_condition *condition =
2285                         lttng_trigger_get_const_condition(trigger);
2286
2287         return lttng_condition_hash(condition);
2288 }
2289
2290 static
2291 int match_trigger(struct cds_lfht_node *node, const void *key)
2292 {
2293         const struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2294         const struct lttng_trigger *trigger = key;
2295
2296         event_notifier_rule = caa_container_of(node,
2297                         const struct ltt_kernel_event_notifier_rule, ht_node);
2298
2299         return lttng_trigger_is_equal(trigger, event_notifier_rule->trigger);
2300 }
2301
2302 static enum lttng_error_code kernel_create_event_notifier_rule(
2303                 struct lttng_trigger *trigger,
2304                 const struct lttng_credentials *creds, uint64_t token)
2305 {
2306         int err, fd, ret = 0;
2307         enum lttng_error_code error_code_ret;
2308         enum lttng_condition_status condition_status;
2309         enum lttng_condition_type condition_type;
2310         enum lttng_event_rule_type event_rule_type;
2311         struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2312         struct lttng_kernel_event_notifier kernel_event_notifier = {};
2313         unsigned int capture_bytecode_count = 0, i;
2314         const struct lttng_condition *condition = NULL;
2315         const struct lttng_event_rule *event_rule = NULL;
2316         enum lttng_condition_status cond_status;
2317
2318         assert(trigger);
2319
2320         condition = lttng_trigger_get_const_condition(trigger);
2321         assert(condition);
2322
2323         condition_type = lttng_condition_get_type(condition);
2324         assert(condition_type == LTTNG_CONDITION_TYPE_ON_EVENT);
2325
2326         /* Does not acquire a reference. */
2327         condition_status = lttng_condition_on_event_get_rule(
2328                         condition, &event_rule);
2329         assert(condition_status == LTTNG_CONDITION_STATUS_OK);
2330         assert(event_rule);
2331
2332         event_rule_type = lttng_event_rule_get_type(event_rule);
2333         assert(event_rule_type != LTTNG_EVENT_RULE_TYPE_UNKNOWN);
2334
2335         error_code_ret = trace_kernel_create_event_notifier_rule(trigger, token,
2336                         lttng_condition_on_event_get_error_counter_index(condition),
2337                         &event_notifier_rule);
2338         if (error_code_ret != LTTNG_OK) {
2339                 goto error;
2340         }
2341
2342         error_code_ret = trace_kernel_init_event_notifier_from_event_rule(
2343                         event_rule, &kernel_event_notifier);
2344         if (error_code_ret != LTTNG_OK) {
2345                 goto free_event;
2346         }
2347
2348         kernel_event_notifier.event.token = event_notifier_rule->token;
2349         kernel_event_notifier.error_counter_idx =
2350                         lttng_condition_on_event_get_error_counter_index(condition);
2351
2352         fd = kernctl_create_event_notifier(
2353                         kernel_tracer_event_notifier_group_fd,
2354                         &kernel_event_notifier);
2355         if (fd < 0) {
2356                 switch (-fd) {
2357                 case EEXIST:
2358                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2359                         break;
2360                 case ENOSYS:
2361                         WARN("Failed to create kernel event notifier: not notifier type not implemented");
2362                         error_code_ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
2363                         break;
2364                 case ENOENT:
2365                         WARN("Failed to create kernel event notifier: not found: name = '%s'",
2366                                         kernel_event_notifier.event.name);
2367                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2368                         break;
2369                 default:
2370                         PERROR("Failed to create kernel event notifier: error code = %d, name = '%s'",
2371                                         fd, kernel_event_notifier.event.name);
2372                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2373                 }
2374                 goto free_event;
2375         }
2376
2377         event_notifier_rule->fd = fd;
2378         /* Prevent fd duplication after execlp(). */
2379         err = fcntl(event_notifier_rule->fd, F_SETFD, FD_CLOEXEC);
2380         if (err < 0) {
2381                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier file descriptor: fd = %d",
2382                                 fd);
2383                 error_code_ret = LTTNG_ERR_FATAL;
2384                 goto set_cloexec_error;
2385         }
2386
2387         if (event_notifier_rule->filter) {
2388                 err = kernctl_filter(event_notifier_rule->fd, event_notifier_rule->filter);
2389                 if (err < 0) {
2390                         switch (-err) {
2391                         case ENOMEM:
2392                                 error_code_ret = LTTNG_ERR_FILTER_NOMEM;
2393                                 break;
2394                         default:
2395                                 error_code_ret = LTTNG_ERR_FILTER_INVAL;
2396                                 break;
2397                         }
2398                         goto filter_error;
2399                 }
2400         }
2401
2402         if (lttng_event_rule_get_type(event_rule) ==
2403                         LTTNG_EVENT_RULE_TYPE_USERSPACE_PROBE) {
2404                 ret = userspace_probe_event_rule_add_callsites(
2405                                 event_rule, creds, event_notifier_rule->fd);
2406                 if (ret) {
2407                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2408                         goto add_callsite_error;
2409                 }
2410         }
2411
2412         /* Set the capture bytecode if any. */
2413         cond_status = lttng_condition_on_event_get_capture_descriptor_count(
2414                         condition, &capture_bytecode_count);
2415         assert(cond_status == LTTNG_CONDITION_STATUS_OK);
2416
2417         for (i = 0; i < capture_bytecode_count; i++) {
2418                 const struct lttng_bytecode *capture_bytecode =
2419                                 lttng_condition_on_event_get_capture_bytecode_at_index(
2420                                                 condition, i);
2421
2422                 if (capture_bytecode == NULL) {
2423                         ERR("Unexpected NULL capture bytecode on condition");
2424                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2425                         goto capture_error;
2426                 }
2427
2428                 ret = kernctl_capture(event_notifier_rule->fd, capture_bytecode);
2429                 if (ret < 0) {
2430                         ERR("Failed to set capture bytecode on event notifier rule fd: fd = %d",
2431                                         event_notifier_rule->fd);
2432                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2433                         goto capture_error;
2434                 }
2435         }
2436
2437         err = kernctl_enable(event_notifier_rule->fd);
2438         if (err < 0) {
2439                 switch (-err) {
2440                 case EEXIST:
2441                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2442                         break;
2443                 default:
2444                         PERROR("enable kernel event notifier");
2445                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2446                         break;
2447                 }
2448                 goto enable_error;
2449         }
2450
2451         /* Add trigger to kernel token mapping in the hash table. */
2452         rcu_read_lock();
2453         cds_lfht_add(kernel_token_to_event_notifier_rule_ht, hash_trigger(trigger),
2454                         &event_notifier_rule->ht_node);
2455         rcu_read_unlock();
2456
2457         DBG("Created kernel event notifier: name = '%s', fd = %d",
2458                         kernel_event_notifier.event.name,
2459                         event_notifier_rule->fd);
2460
2461         return LTTNG_OK;
2462
2463 capture_error:
2464 add_callsite_error:
2465 enable_error:
2466 set_cloexec_error:
2467 filter_error:
2468         {
2469                 const int close_ret = close(event_notifier_rule->fd);
2470
2471                 if (close_ret) {
2472                         PERROR("Failed to close kernel event notifier file descriptor: fd = %d",
2473                                         event_notifier_rule->fd);
2474                 }
2475         }
2476 free_event:
2477         free(event_notifier_rule);
2478 error:
2479         return error_code_ret;
2480 }
2481
2482 enum lttng_error_code kernel_register_event_notifier(
2483                 struct lttng_trigger *trigger,
2484                 const struct lttng_credentials *cmd_creds)
2485 {
2486         enum lttng_error_code ret;
2487         enum lttng_condition_status status;
2488         enum lttng_domain_type domain_type;
2489         const struct lttng_event_rule *event_rule;
2490         const struct lttng_condition *const condition =
2491                         lttng_trigger_get_const_condition(trigger);
2492         const uint64_t token = lttng_trigger_get_tracer_token(trigger);
2493
2494         assert(condition);
2495
2496         /* Does not acquire a reference to the event rule. */
2497         status = lttng_condition_on_event_get_rule(
2498                         condition, &event_rule);
2499         assert(status == LTTNG_CONDITION_STATUS_OK);
2500
2501         domain_type = lttng_event_rule_get_domain_type(event_rule);
2502         assert(domain_type == LTTNG_DOMAIN_KERNEL);
2503
2504         ret = kernel_create_event_notifier_rule(trigger, cmd_creds, token);
2505         if (ret != LTTNG_OK) {
2506                 ERR("Failed to create kernel event notifier rule");
2507         }
2508
2509         return ret;
2510 }
2511
2512 enum lttng_error_code kernel_unregister_event_notifier(
2513                 const struct lttng_trigger *trigger)
2514 {
2515         struct ltt_kernel_event_notifier_rule *token_event_rule_element;
2516         struct cds_lfht_node *node;
2517         struct cds_lfht_iter iter;
2518         enum lttng_error_code error_code_ret;
2519         int ret;
2520
2521         rcu_read_lock();
2522
2523         cds_lfht_lookup(kernel_token_to_event_notifier_rule_ht,
2524                         hash_trigger(trigger), match_trigger, trigger, &iter);
2525
2526         node = cds_lfht_iter_get_node(&iter);
2527         if (!node) {
2528                 error_code_ret = LTTNG_ERR_TRIGGER_NOT_FOUND;
2529                 goto error;
2530         }
2531
2532         token_event_rule_element = caa_container_of(node,
2533                         struct ltt_kernel_event_notifier_rule, ht_node);
2534
2535         ret = kernel_disable_event_notifier_rule(token_event_rule_element);
2536         if (ret) {
2537                 error_code_ret = LTTNG_ERR_FATAL;
2538                 goto error;
2539         }
2540
2541         trace_kernel_destroy_event_notifier_rule(token_event_rule_element);
2542         error_code_ret = LTTNG_OK;
2543
2544 error:
2545         rcu_read_unlock();
2546
2547         return error_code_ret;
2548 }
2549
2550 int kernel_get_notification_fd(void)
2551 {
2552         return kernel_tracer_event_notifier_group_notification_fd;
2553 }