src/bin/lttng-sessiond/kernel.c

   1 /*
   2  * Copyright (C) 2011 - David Goulet <david.goulet@polymtl.ca>
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License, version 2 only,
   6  * as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16  */
  17
  18 #define _LGPL_SOURCE
  19 #include <fcntl.h>
  20 #include <stdlib.h>
  21 #include <stdio.h>
  22 #include <string.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <common/common.h>
  27 #include <common/kernel-ctl/kernel-ctl.h>
  28 #include <common/kernel-ctl/kernel-ioctl.h>
  29 #include <common/sessiond-comm/sessiond-comm.h>
  30
  31 #include "consumer.h"
  32 #include "kernel.h"
  33 #include "kernel-consumer.h"
  34 #include "kern-modules.h"
  35 #include "utils.h"
  36 #include "rotate.h"
  37
  38 /*
  39  * Key used to reference a channel between the sessiond and the consumer. This
  40  * is only read and updated with the session_list lock held.
  41  */
  42 static uint64_t next_kernel_channel_key;
  43
  44 #include <lttng/userspace-probe.h>
  45 #include <lttng/userspace-probe-internal.h>
  46 /*
  47  * Add context on a kernel channel.
  48  *
  49  * Assumes the ownership of ctx.
  50  */
  51 int kernel_add_channel_context(struct ltt_kernel_channel *chan,
  52                 struct ltt_kernel_context *ctx)
  53 {
  54         int ret;
  55
  56         assert(chan);
  57         assert(ctx);
  58
  59         DBG("Adding context to channel %s", chan->channel->name);
  60         ret = kernctl_add_context(chan->fd, &ctx->ctx);
  61         if (ret < 0) {
  62                 switch (-ret) {
  63                 case ENOSYS:
  64                         /* Exists but not available for this kernel */
  65                         ret = LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE;
  66                         goto error;
  67                 case EEXIST:
  68                         /* If EEXIST, we just ignore the error */
  69                         ret = 0;
  70                         goto end;
  71                 default:
  72                         PERROR("add context ioctl");
  73                         ret = LTTNG_ERR_KERN_CONTEXT_FAIL;
  74                         goto error;
  75                 }
  76         }
  77         ret = 0;
  78
  79 end:
  80         cds_list_add_tail(&ctx->list, &chan->ctx_list);
  81         ctx->in_list = true;
  82         ctx = NULL;
  83 error:
  84         if (ctx) {
  85                 trace_kernel_destroy_context(ctx);
  86         }
  87         return ret;
  88 }
  89
  90 /*
  91  * Create a new kernel session, register it to the kernel tracer and add it to
  92  * the session daemon session.
  93  */
  94 int kernel_create_session(struct ltt_session *session, int tracer_fd)
  95 {
  96         int ret;
  97         struct ltt_kernel_session *lks;
  98
  99         assert(session);
 100
 101         /* Allocate data structure */
 102         lks = trace_kernel_create_session();
 103         if (lks == NULL) {
 104                 ret = -1;
 105                 goto error;
 106         }
 107
 108         /* Kernel tracer session creation */
 109         ret = kernctl_create_session(tracer_fd);
 110         if (ret < 0) {
 111                 PERROR("ioctl kernel create session");
 112                 goto error;
 113         }
 114
 115         lks->fd = ret;
 116         /* Prevent fd duplication after execlp() */
 117         ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
 118         if (ret < 0) {
 119                 PERROR("fcntl session fd");
 120         }
 121
 122         lks->id = session->id;
 123         lks->consumer_fds_sent = 0;
 124         session->kernel_session = lks;
 125
 126         DBG("Kernel session created (fd: %d)", lks->fd);
 127
 128         return 0;
 129
 130 error:
 131         if (lks) {
 132                 trace_kernel_destroy_session(lks);
 133         }
 134         return ret;
 135 }
 136
 137 /*
 138  * Create a kernel channel, register it to the kernel tracer and add it to the
 139  * kernel session.
 140  */
 141 int kernel_create_channel(struct ltt_kernel_session *session,
 142                 struct lttng_channel *chan)
 143 {
 144         int ret;
 145         struct ltt_kernel_channel *lkc;
 146
 147         assert(session);
 148         assert(chan);
 149
 150         /* Allocate kernel channel */
 151         lkc = trace_kernel_create_channel(chan);
 152         if (lkc == NULL) {
 153                 goto error;
 154         }
 155
 156         DBG3("Kernel create channel %s with attr: %d, %" PRIu64 ", %" PRIu64 ", %u, %u, %d, %d",
 157                         chan->name, lkc->channel->attr.overwrite,
 158                         lkc->channel->attr.subbuf_size, lkc->channel->attr.num_subbuf,
 159                         lkc->channel->attr.switch_timer_interval, lkc->channel->attr.read_timer_interval,
 160                         lkc->channel->attr.live_timer_interval, lkc->channel->attr.output);
 161
 162         /* Kernel tracer channel creation */
 163         ret = kernctl_create_channel(session->fd, &lkc->channel->attr);
 164         if (ret < 0) {
 165                 PERROR("ioctl kernel create channel");
 166                 goto error;
 167         }
 168
 169         /* Setup the channel fd */
 170         lkc->fd = ret;
 171         /* Prevent fd duplication after execlp() */
 172         ret = fcntl(lkc->fd, F_SETFD, FD_CLOEXEC);
 173         if (ret < 0) {
 174                 PERROR("fcntl session fd");
 175         }
 176
 177         /* Add channel to session */
 178         cds_list_add(&lkc->list, &session->channel_list.head);
 179         session->channel_count++;
 180         lkc->session = session;
 181         lkc->key = ++next_kernel_channel_key;
 182
 183         DBG("Kernel channel %s created (fd: %d, key: %" PRIu64 ")",
 184                         lkc->channel->name, lkc->fd, lkc->key);
 185
 186         return 0;
 187
 188 error:
 189         if (lkc) {
 190                 free(lkc->channel);
 191                 free(lkc);
 192         }
 193         return -1;
 194 }
 195
 196 /*
 197  * Compute the offset of the instrumentation byte in the binary based on the
 198  * function probe location using the ELF lookup method.
 199  *
 200  * Returns 0 on success and set the offset out parameter to the offset of the
 201  * elf symbol
 202  * Returns -1 on error
 203  */
 204 static
 205 int extract_userspace_probe_offset_function_elf(
 206                 struct lttng_userspace_probe_location *probe_location,
 207                 struct ltt_kernel_session *session, uint64_t *offset)
 208 {
 209         int fd;
 210         int ret = 0;
 211         const char *symbol = NULL;
 212         struct lttng_userspace_probe_location_lookup_method *lookup = NULL;
 213         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 214
 215
 216         assert(lttng_userspace_probe_location_get_type(probe_location) ==
 217                         LTTNG_USERSPACE_PROBE_LOCATION_TYPE_FUNCTION);
 218
 219         lookup = lttng_userspace_probe_location_get_lookup_method(
 220                         probe_location);
 221         if (!lookup) {
 222                 ret = -1;
 223                 goto end;
 224         }
 225
 226         lookup_method_type =
 227                         lttng_userspace_probe_location_lookup_method_get_type(lookup);
 228
 229         assert(lookup_method_type ==
 230                         LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF);
 231
 232         symbol = lttng_userspace_probe_location_function_get_function_name(
 233                         probe_location);
 234         if (!symbol) {
 235                 ret = -1;
 236                 goto end;
 237         }
 238
 239         fd = lttng_userspace_probe_location_function_get_binary_fd(probe_location);
 240         if (fd < 0) {
 241                 ret = -1;
 242                 goto end;
 243         }
 244
 245         ret = run_as_extract_elf_symbol_offset(fd, symbol, session->uid,
 246                         session->gid, offset);
 247         if (ret < 0) {
 248                 DBG("userspace probe offset calculation failed for "
 249                                 "function %s", symbol);
 250                 goto end;
 251         }
 252
 253         DBG("userspace probe elf offset for %s is 0x%jd", symbol, (intmax_t)(*offset));
 254 end:
 255         return ret;
 256 }
 257
 258 /*
 259  * Compute the offsets of the instrumentation bytes in the binary based on the
 260  * tracepoint probe location using the SDT lookup method. This function
 261  * allocates the offsets buffer, the caller must free it.
 262  *
 263  * Returns 0 on success and set the offset out parameter to the offsets of the
 264  * SDT tracepoint.
 265  * Returns -1 on error.
 266  */
 267 static
 268 int extract_userspace_probe_offset_tracepoint_sdt(
 269                 struct lttng_userspace_probe_location *probe_location,
 270                 struct ltt_kernel_session *session, uint64_t **offsets,
 271                 uint32_t *offsets_count)
 272 {
 273         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 274         struct lttng_userspace_probe_location_lookup_method *lookup = NULL;
 275         const char *probe_name = NULL, *provider_name = NULL;
 276         int ret = 0;
 277         int fd, i;
 278
 279         assert(lttng_userspace_probe_location_get_type(probe_location) ==
 280                         LTTNG_USERSPACE_PROBE_LOCATION_TYPE_TRACEPOINT);
 281
 282         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 283         if (!lookup) {
 284                 ret = -1;
 285                 goto end;
 286         }
 287
 288         lookup_method_type =
 289                         lttng_userspace_probe_location_lookup_method_get_type(lookup);
 290
 291         assert(lookup_method_type ==
 292                         LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT);
 293
 294
 295         probe_name = lttng_userspace_probe_location_tracepoint_get_probe_name(
 296                         probe_location);
 297         if (!probe_name) {
 298                 ret = -1;
 299                 goto end;
 300         }
 301
 302         provider_name = lttng_userspace_probe_location_tracepoint_get_provider_name(
 303                         probe_location);
 304         if (!provider_name) {
 305                 ret = -1;
 306                 goto end;
 307         }
 308
 309         fd = lttng_userspace_probe_location_tracepoint_get_binary_fd(probe_location);
 310         if (fd < 0) {
 311                 ret = -1;
 312                 goto end;
 313         }
 314
 315         ret = run_as_extract_sdt_probe_offsets(fd, provider_name, probe_name,
 316                         session->uid, session->gid, offsets, offsets_count);
 317         if (ret < 0) {
 318                 DBG("userspace probe offset calculation failed for sdt "
 319                                 "probe %s:%s", provider_name, probe_name);
 320                 goto end;
 321         }
 322
 323         if (*offsets_count == 0) {
 324                 DBG("no userspace probe offset found");
 325                 goto end;
 326         }
 327
 328         DBG("%u userspace probe SDT offsets found for %s:%s at:",
 329                         *offsets_count, provider_name, probe_name);
 330         for (i = 0; i < *offsets_count; i++) {
 331                 DBG("\t0x%jd", (intmax_t)((*offsets)[i]));
 332         }
 333 end:
 334         return ret;
 335 }
 336
 337 /*
 338  * Extract the offsets of the instrumentation point for the different lookup
 339  * methods.
 340  */
 341 static
 342 int userspace_probe_add_callsites(struct lttng_event *ev,
 343                         struct ltt_kernel_session *session, int fd)
 344 {
 345         struct lttng_userspace_probe_location_lookup_method *lookup_method = NULL;
 346         enum lttng_userspace_probe_location_lookup_method_type type;
 347         struct lttng_userspace_probe_location *location = NULL;
 348         int ret;
 349
 350         assert(ev);
 351         assert(ev->type == LTTNG_EVENT_USERSPACE_PROBE);
 352
 353         location = lttng_event_get_userspace_probe_location(ev);
 354         if (!location) {
 355                 ret = -1;
 356                 goto end;
 357         }
 358         lookup_method =
 359                         lttng_userspace_probe_location_get_lookup_method(location);
 360         if (!lookup_method) {
 361                 ret = -1;
 362                 goto end;
 363         }
 364
 365         type = lttng_userspace_probe_location_lookup_method_get_type(lookup_method);
 366         switch (type) {
 367         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF:
 368         {
 369                 struct lttng_kernel_event_callsite callsite;
 370                 uint64_t offset;
 371
 372                 ret = extract_userspace_probe_offset_function_elf(location, session, &offset);
 373                 if (ret) {
 374                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 375                         goto end;
 376                 }
 377
 378                 callsite.u.uprobe.offset = offset;
 379                 ret = kernctl_add_callsite(fd, &callsite);
 380                 if (ret) {
 381                         WARN("Adding callsite to userspace probe "
 382                                         "event %s failed.", ev->name);
 383                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 384                         goto end;
 385                 }
 386                 break;
 387         }
 388         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT:
 389         {
 390                 int i;
 391                 uint64_t *offsets = NULL;
 392                 uint32_t offsets_count;
 393                 struct lttng_kernel_event_callsite callsite;
 394
 395                 /*
 396                  * This call allocates the offsets buffer. This buffer must be freed
 397                  * by the caller
 398                  */
 399                 ret = extract_userspace_probe_offset_tracepoint_sdt(location, session,
 400                                 &offsets, &offsets_count);
 401                 if (ret) {
 402                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 403                         goto end;
 404                 }
 405                 for (i = 0; i < offsets_count; i++) {
 406                         callsite.u.uprobe.offset = offsets[i];
 407                         ret = kernctl_add_callsite(fd, &callsite);
 408                         if (ret) {
 409                                 WARN("Adding callsite to userspace probe "
 410                                                 "event %s failed.", ev->name);
 411                                 ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 412                                 free(offsets);
 413                                 goto end;
 414                         }
 415                 }
 416                 free(offsets);
 417                 break;
 418         }
 419         default:
 420                 ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 421                 goto end;
 422         }
 423 end:
 424         return ret;
 425 }
 426
 427 /*
 428  * Create a kernel event, enable it to the kernel tracer and add it to the
 429  * channel event list of the kernel session.
 430  * We own filter_expression and filter.
 431  */
 432 int kernel_create_event(struct lttng_event *ev,
 433                 struct ltt_kernel_channel *channel,
 434                 char *filter_expression,
 435                 struct lttng_filter_bytecode *filter)
 436 {
 437         int err, fd;
 438         enum lttng_error_code ret;
 439         struct ltt_kernel_event *event;
 440
 441         assert(ev);
 442         assert(channel);
 443
 444         /* We pass ownership of filter_expression and filter */
 445         ret = trace_kernel_create_event(ev, filter_expression,
 446                         filter, &event);
 447         if (ret != LTTNG_OK) {
 448                 goto error;
 449         }
 450
 451         fd = kernctl_create_event(channel->fd, event->event);
 452         if (fd < 0) {
 453                 switch (-fd) {
 454                 case EEXIST:
 455                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 456                         break;
 457                 case ENOSYS:
 458                         WARN("Event type not implemented");
 459                         ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
 460                         break;
 461                 case ENOENT:
 462                         WARN("Event %s not found!", ev->name);
 463                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 464                         break;
 465                 default:
 466                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 467                         PERROR("create event ioctl");
 468                 }
 469                 goto free_event;
 470         }
 471
 472         event->type = ev->type;
 473         event->fd = fd;
 474         /* Prevent fd duplication after execlp() */
 475         err = fcntl(event->fd, F_SETFD, FD_CLOEXEC);
 476         if (err < 0) {
 477                 PERROR("fcntl session fd");
 478         }
 479
 480         if (filter) {
 481                 err = kernctl_filter(event->fd, filter);
 482                 if (err < 0) {
 483                         switch (-err) {
 484                         case ENOMEM:
 485                                 ret = LTTNG_ERR_FILTER_NOMEM;
 486                                 break;
 487                         default:
 488                                 ret = LTTNG_ERR_FILTER_INVAL;
 489                                 break;
 490                         }
 491                         goto filter_error;
 492                 }
 493         }
 494
 495         if (ev->type == LTTNG_EVENT_USERSPACE_PROBE) {
 496                 ret = userspace_probe_add_callsites(ev, channel->session, event->fd);
 497                 if (ret) {
 498                         goto add_callsite_error;
 499                 }
 500         }
 501
 502         err = kernctl_enable(event->fd);
 503         if (err < 0) {
 504                 switch (-err) {
 505                 case EEXIST:
 506                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 507                         break;
 508                 default:
 509                         PERROR("enable kernel event");
 510                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 511                         break;
 512                 }
 513                 goto enable_error;
 514         }
 515
 516         /* Add event to event list */
 517         cds_list_add(&event->list, &channel->events_list.head);
 518         channel->event_count++;
 519
 520         DBG("Event %s created (fd: %d)", ev->name, event->fd);
 521
 522         return 0;
 523
 524 add_callsite_error:
 525 enable_error:
 526 filter_error:
 527         {
 528                 int closeret;
 529
 530                 closeret = close(event->fd);
 531                 if (closeret) {
 532                         PERROR("close event fd");
 533                 }
 534         }
 535 free_event:
 536         free(event);
 537 error:
 538         return ret;
 539 }
 540
 541 /*
 542  * Disable a kernel channel.
 543  */
 544 int kernel_disable_channel(struct ltt_kernel_channel *chan)
 545 {
 546         int ret;
 547
 548         assert(chan);
 549
 550         ret = kernctl_disable(chan->fd);
 551         if (ret < 0) {
 552                 PERROR("disable chan ioctl");
 553                 goto error;
 554         }
 555
 556         chan->enabled = 0;
 557         DBG("Kernel channel %s disabled (fd: %d, key: %" PRIu64 ")",
 558                         chan->channel->name, chan->fd, chan->key);
 559
 560         return 0;
 561
 562 error:
 563         return ret;
 564 }
 565
 566 /*
 567  * Enable a kernel channel.
 568  */
 569 int kernel_enable_channel(struct ltt_kernel_channel *chan)
 570 {
 571         int ret;
 572
 573         assert(chan);
 574
 575         ret = kernctl_enable(chan->fd);
 576         if (ret < 0 && ret != -EEXIST) {
 577                 PERROR("Enable kernel chan");
 578                 goto error;
 579         }
 580
 581         chan->enabled = 1;
 582         DBG("Kernel channel %s enabled (fd: %d, key: %" PRIu64 ")",
 583                         chan->channel->name, chan->fd, chan->key);
 584
 585         return 0;
 586
 587 error:
 588         return ret;
 589 }
 590
 591 /*
 592  * Enable a kernel event.
 593  */
 594 int kernel_enable_event(struct ltt_kernel_event *event)
 595 {
 596         int ret;
 597
 598         assert(event);
 599
 600         ret = kernctl_enable(event->fd);
 601         if (ret < 0) {
 602                 switch (-ret) {
 603                 case EEXIST:
 604                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 605                         break;
 606                 default:
 607                         PERROR("enable kernel event");
 608                         break;
 609                 }
 610                 goto error;
 611         }
 612
 613         event->enabled = 1;
 614         DBG("Kernel event %s enabled (fd: %d)", event->event->name, event->fd);
 615
 616         return 0;
 617
 618 error:
 619         return ret;
 620 }
 621
 622 /*
 623  * Disable a kernel event.
 624  */
 625 int kernel_disable_event(struct ltt_kernel_event *event)
 626 {
 627         int ret;
 628
 629         assert(event);
 630
 631         ret = kernctl_disable(event->fd);
 632         if (ret < 0) {
 633                 switch (-ret) {
 634                 case EEXIST:
 635                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 636                         break;
 637                 default:
 638                         PERROR("disable kernel event");
 639                         break;
 640                 }
 641                 goto error;
 642         }
 643
 644         event->enabled = 0;
 645         DBG("Kernel event %s disabled (fd: %d)", event->event->name, event->fd);
 646
 647         return 0;
 648
 649 error:
 650         return ret;
 651 }
 652
 653
 654 int kernel_track_pid(struct ltt_kernel_session *session, int pid)
 655 {
 656         int ret;
 657
 658         DBG("Kernel track PID %d for session id %" PRIu64 ".",
 659                         pid, session->id);
 660         ret = kernctl_track_pid(session->fd, pid);
 661         if (!ret) {
 662                 return LTTNG_OK;
 663         }
 664         switch (-ret) {
 665         case EINVAL:
 666                 return LTTNG_ERR_INVALID;
 667         case ENOMEM:
 668                 return LTTNG_ERR_NOMEM;
 669         case EEXIST:
 670                 return LTTNG_ERR_PID_TRACKED;
 671         default:
 672                 return LTTNG_ERR_UNK;
 673         }
 674 }
 675
 676 int kernel_untrack_pid(struct ltt_kernel_session *session, int pid)
 677 {
 678         int ret;
 679
 680         DBG("Kernel untrack PID %d for session id %" PRIu64 ".",
 681                         pid, session->id);
 682         ret = kernctl_untrack_pid(session->fd, pid);
 683         if (!ret) {
 684                 return LTTNG_OK;
 685         }
 686         switch (-ret) {
 687         case EINVAL:
 688                 return LTTNG_ERR_INVALID;
 689         case ENOMEM:
 690                 return LTTNG_ERR_NOMEM;
 691         case ENOENT:
 692                 return LTTNG_ERR_PID_NOT_TRACKED;
 693         default:
 694                 return LTTNG_ERR_UNK;
 695         }
 696 }
 697
 698 ssize_t kernel_list_tracker_pids(struct ltt_kernel_session *session,
 699                 int **_pids)
 700 {
 701         int fd, ret;
 702         int pid;
 703         ssize_t nbmem, count = 0;
 704         FILE *fp;
 705         int *pids;
 706
 707         fd = kernctl_list_tracker_pids(session->fd);
 708         if (fd < 0) {
 709                 PERROR("kernel tracker pids list");
 710                 goto error;
 711         }
 712
 713         fp = fdopen(fd, "r");
 714         if (fp == NULL) {
 715                 PERROR("kernel tracker pids list fdopen");
 716                 goto error_fp;
 717         }
 718
 719         nbmem = KERNEL_TRACKER_PIDS_INIT_LIST_SIZE;
 720         pids = zmalloc(sizeof(*pids) * nbmem);
 721         if (pids == NULL) {
 722                 PERROR("alloc list pids");
 723                 count = -ENOMEM;
 724                 goto end;
 725         }
 726
 727         while (fscanf(fp, "process { pid = %u; };\n", &pid) == 1) {
 728                 if (count >= nbmem) {
 729                         int *new_pids;
 730                         size_t new_nbmem;
 731
 732                         new_nbmem = nbmem << 1;
 733                         DBG("Reallocating pids list from %zu to %zu entries",
 734                                         nbmem, new_nbmem);
 735                         new_pids = realloc(pids, new_nbmem * sizeof(*new_pids));
 736                         if (new_pids == NULL) {
 737                                 PERROR("realloc list events");
 738                                 free(pids);
 739                                 count = -ENOMEM;
 740                                 goto end;
 741                         }
 742                         /* Zero the new memory */
 743                         memset(new_pids + nbmem, 0,
 744                                 (new_nbmem - nbmem) * sizeof(*new_pids));
 745                         nbmem = new_nbmem;
 746                         pids = new_pids;
 747                 }
 748                 pids[count++] = pid;
 749         }
 750
 751         *_pids = pids;
 752         DBG("Kernel list tracker pids done (%zd pids)", count);
 753 end:
 754         ret = fclose(fp);       /* closes both fp and fd */
 755         if (ret) {
 756                 PERROR("fclose");
 757         }
 758         return count;
 759
 760 error_fp:
 761         ret = close(fd);
 762         if (ret) {
 763                 PERROR("close");
 764         }
 765 error:
 766         return -1;
 767 }
 768
 769 /*
 770  * Create kernel metadata, open from the kernel tracer and add it to the
 771  * kernel session.
 772  */
 773 int kernel_open_metadata(struct ltt_kernel_session *session)
 774 {
 775         int ret;
 776         struct ltt_kernel_metadata *lkm = NULL;
 777
 778         assert(session);
 779
 780         /* Allocate kernel metadata */
 781         lkm = trace_kernel_create_metadata();
 782         if (lkm == NULL) {
 783                 goto error;
 784         }
 785
 786         /* Kernel tracer metadata creation */
 787         ret = kernctl_open_metadata(session->fd, &lkm->conf->attr);
 788         if (ret < 0) {
 789                 goto error_open;
 790         }
 791
 792         lkm->fd = ret;
 793         lkm->key = ++next_kernel_channel_key;
 794         /* Prevent fd duplication after execlp() */
 795         ret = fcntl(lkm->fd, F_SETFD, FD_CLOEXEC);
 796         if (ret < 0) {
 797                 PERROR("fcntl session fd");
 798         }
 799
 800         session->metadata = lkm;
 801
 802         DBG("Kernel metadata opened (fd: %d)", lkm->fd);
 803
 804         return 0;
 805
 806 error_open:
 807         trace_kernel_destroy_metadata(lkm);
 808 error:
 809         return -1;
 810 }
 811
 812 /*
 813  * Start tracing session.
 814  */
 815 int kernel_start_session(struct ltt_kernel_session *session)
 816 {
 817         int ret;
 818
 819         assert(session);
 820
 821         ret = kernctl_start_session(session->fd);
 822         if (ret < 0) {
 823                 PERROR("ioctl start session");
 824                 goto error;
 825         }
 826
 827         DBG("Kernel session started");
 828
 829         return 0;
 830
 831 error:
 832         return ret;
 833 }
 834
 835 /*
 836  * Make a kernel wait to make sure in-flight probe have completed.
 837  */
 838 void kernel_wait_quiescent(int fd)
 839 {
 840         int ret;
 841
 842         DBG("Kernel quiescent wait on %d", fd);
 843
 844         ret = kernctl_wait_quiescent(fd);
 845         if (ret < 0) {
 846                 PERROR("wait quiescent ioctl");
 847                 ERR("Kernel quiescent wait failed");
 848         }
 849 }
 850
 851 /*
 852  *  Force flush buffer of metadata.
 853  */
 854 int kernel_metadata_flush_buffer(int fd)
 855 {
 856         int ret;
 857
 858         DBG("Kernel flushing metadata buffer on fd %d", fd);
 859
 860         ret = kernctl_buffer_flush(fd);
 861         if (ret < 0) {
 862                 ERR("Fail to flush metadata buffers %d (ret: %d)", fd, ret);
 863         }
 864
 865         return 0;
 866 }
 867
 868 /*
 869  * Force flush buffer for channel.
 870  */
 871 int kernel_flush_buffer(struct ltt_kernel_channel *channel)
 872 {
 873         int ret;
 874         struct ltt_kernel_stream *stream;
 875
 876         assert(channel);
 877
 878         DBG("Flush buffer for channel %s", channel->channel->name);
 879
 880         cds_list_for_each_entry(stream, &channel->stream_list.head, list) {
 881                 DBG("Flushing channel stream %d", stream->fd);
 882                 ret = kernctl_buffer_flush(stream->fd);
 883                 if (ret < 0) {
 884                         PERROR("ioctl");
 885                         ERR("Fail to flush buffer for stream %d (ret: %d)",
 886                                         stream->fd, ret);
 887                 }
 888         }
 889
 890         return 0;
 891 }
 892
 893 /*
 894  * Stop tracing session.
 895  */
 896 int kernel_stop_session(struct ltt_kernel_session *session)
 897 {
 898         int ret;
 899
 900         assert(session);
 901
 902         ret = kernctl_stop_session(session->fd);
 903         if (ret < 0) {
 904                 goto error;
 905         }
 906
 907         DBG("Kernel session stopped");
 908
 909         return 0;
 910
 911 error:
 912         return ret;
 913 }
 914
 915 /*
 916  * Open stream of channel, register it to the kernel tracer and add it
 917  * to the stream list of the channel.
 918  *
 919  * Note: given that the streams may appear in random order wrt CPU
 920  * number (e.g. cpu hotplug), the index value of the stream number in
 921  * the stream name is not necessarily linked to the CPU number.
 922  *
 923  * Return the number of created stream. Else, a negative value.
 924  */
 925 int kernel_open_channel_stream(struct ltt_kernel_channel *channel)
 926 {
 927         int ret;
 928         struct ltt_kernel_stream *lks;
 929
 930         assert(channel);
 931
 932         while ((ret = kernctl_create_stream(channel->fd)) >= 0) {
 933                 lks = trace_kernel_create_stream(channel->channel->name,
 934                                 channel->stream_count);
 935                 if (lks == NULL) {
 936                         ret = close(ret);
 937                         if (ret) {
 938                                 PERROR("close");
 939                         }
 940                         goto error;
 941                 }
 942
 943                 lks->fd = ret;
 944                 /* Prevent fd duplication after execlp() */
 945                 ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
 946                 if (ret < 0) {
 947                         PERROR("fcntl session fd");
 948                 }
 949
 950                 lks->tracefile_size = channel->channel->attr.tracefile_size;
 951                 lks->tracefile_count = channel->channel->attr.tracefile_count;
 952
 953                 /* Add stream to channel stream list */
 954                 cds_list_add(&lks->list, &channel->stream_list.head);
 955                 channel->stream_count++;
 956
 957                 DBG("Kernel stream %s created (fd: %d, state: %d)", lks->name, lks->fd,
 958                                 lks->state);
 959         }
 960
 961         return channel->stream_count;
 962
 963 error:
 964         return -1;
 965 }
 966
 967 /*
 968  * Open the metadata stream and set it to the kernel session.
 969  */
 970 int kernel_open_metadata_stream(struct ltt_kernel_session *session)
 971 {
 972         int ret;
 973
 974         assert(session);
 975
 976         ret = kernctl_create_stream(session->metadata->fd);
 977         if (ret < 0) {
 978                 PERROR("kernel create metadata stream");
 979                 goto error;
 980         }
 981
 982         DBG("Kernel metadata stream created (fd: %d)", ret);
 983         session->metadata_stream_fd = ret;
 984         /* Prevent fd duplication after execlp() */
 985         ret = fcntl(session->metadata_stream_fd, F_SETFD, FD_CLOEXEC);
 986         if (ret < 0) {
 987                 PERROR("fcntl session fd");
 988         }
 989
 990         return 0;
 991
 992 error:
 993         return -1;
 994 }
 995
 996 /*
 997  * Get the event list from the kernel tracer and return the number of elements.
 998  */
 999 ssize_t kernel_list_events(int tracer_fd, struct lttng_event **events)
1000 {
1001         int fd, ret;
1002         char *event;
1003         size_t nbmem, count = 0;
1004         FILE *fp;
1005         struct lttng_event *elist;
1006
1007         assert(events);
1008
1009         fd = kernctl_tracepoint_list(tracer_fd);
1010         if (fd < 0) {
1011                 PERROR("kernel tracepoint list");
1012                 goto error;
1013         }
1014
1015         fp = fdopen(fd, "r");
1016         if (fp == NULL) {
1017                 PERROR("kernel tracepoint list fdopen");
1018                 goto error_fp;
1019         }
1020
1021         /*
1022          * Init memory size counter
1023          * See kernel-ctl.h for explanation of this value
1024          */
1025         nbmem = KERNEL_EVENT_INIT_LIST_SIZE;
1026         elist = zmalloc(sizeof(struct lttng_event) * nbmem);
1027         if (elist == NULL) {
1028                 PERROR("alloc list events");
1029                 count = -ENOMEM;
1030                 goto end;
1031         }
1032
1033         while (fscanf(fp, "event { name = %m[^;]; };\n", &event) == 1) {
1034                 if (count >= nbmem) {
1035                         struct lttng_event *new_elist;
1036                         size_t new_nbmem;
1037
1038                         new_nbmem = nbmem << 1;
1039                         DBG("Reallocating event list from %zu to %zu bytes",
1040                                         nbmem, new_nbmem);
1041                         new_elist = realloc(elist, new_nbmem * sizeof(struct lttng_event));
1042                         if (new_elist == NULL) {
1043                                 PERROR("realloc list events");
1044                                 free(event);
1045                                 free(elist);
1046                                 count = -ENOMEM;
1047                                 goto end;
1048                         }
1049                         /* Zero the new memory */
1050                         memset(new_elist + nbmem, 0,
1051                                 (new_nbmem - nbmem) * sizeof(struct lttng_event));
1052                         nbmem = new_nbmem;
1053                         elist = new_elist;
1054                 }
1055                 strncpy(elist[count].name, event, LTTNG_SYMBOL_NAME_LEN);
1056                 elist[count].name[LTTNG_SYMBOL_NAME_LEN - 1] = '\0';
1057                 elist[count].enabled = -1;
1058                 count++;
1059                 free(event);
1060         }
1061
1062         *events = elist;
1063         DBG("Kernel list events done (%zu events)", count);
1064 end:
1065         ret = fclose(fp);       /* closes both fp and fd */
1066         if (ret) {
1067                 PERROR("fclose");
1068         }
1069         return count;
1070
1071 error_fp:
1072         ret = close(fd);
1073         if (ret) {
1074                 PERROR("close");
1075         }
1076 error:
1077         return -1;
1078 }
1079
1080 /*
1081  * Get kernel version and validate it.
1082  */
1083 int kernel_validate_version(int tracer_fd,
1084                 struct lttng_kernel_tracer_version *version,
1085                 struct lttng_kernel_tracer_abi_version *abi_version)
1086 {
1087         int ret;
1088
1089         ret = kernctl_tracer_version(tracer_fd, version);
1090         if (ret < 0) {
1091                 ERR("Failed to retrieve the lttng-modules version");
1092                 goto error;
1093         }
1094
1095         /* Validate version */
1096         if (version->major != VERSION_MAJOR) {
1097                 ERR("Kernel tracer major version (%d) is not compatible with lttng-tools major version (%d)",
1098                         version->major, VERSION_MAJOR);
1099                 goto error_version;
1100         }
1101         ret = kernctl_tracer_abi_version(tracer_fd, abi_version);
1102         if (ret < 0) {
1103                 ERR("Failed to retrieve lttng-modules ABI version");
1104                 goto error;
1105         }
1106         if (abi_version->major != LTTNG_MODULES_ABI_MAJOR_VERSION) {
1107                 ERR("Kernel tracer ABI version (%d.%d) does not match the expected ABI major version (%d.*)",
1108                         abi_version->major, abi_version->minor,
1109                         LTTNG_MODULES_ABI_MAJOR_VERSION);
1110                 goto error;
1111         }
1112         DBG2("Kernel tracer version validated (%d.%d, ABI %d.%d)",
1113                         version->major, version->minor,
1114                         abi_version->major, abi_version->minor);
1115         return 0;
1116
1117 error_version:
1118         ret = -1;
1119
1120 error:
1121         ERR("Kernel tracer version check failed; kernel tracing will not be available");
1122         return ret;
1123 }
1124
1125 /*
1126  * Kernel work-arounds called at the start of sessiond main().
1127  */
1128 int init_kernel_workarounds(void)
1129 {
1130         int ret;
1131         FILE *fp;
1132
1133         /*
1134          * boot_id needs to be read once before being used concurrently
1135          * to deal with a Linux kernel race. A fix is proposed for
1136          * upstream, but the work-around is needed for older kernels.
1137          */
1138         fp = fopen("/proc/sys/kernel/random/boot_id", "r");
1139         if (!fp) {
1140                 goto end_boot_id;
1141         }
1142         while (!feof(fp)) {
1143                 char buf[37] = "";
1144
1145                 ret = fread(buf, 1, sizeof(buf), fp);
1146                 if (ret < 0) {
1147                         /* Ignore error, we don't really care */
1148                 }
1149         }
1150         ret = fclose(fp);
1151         if (ret) {
1152                 PERROR("fclose");
1153         }
1154 end_boot_id:
1155         return 0;
1156 }
1157
1158 /*
1159  * Complete teardown of a kernel session.
1160  */
1161 void kernel_destroy_session(struct ltt_kernel_session *ksess)
1162 {
1163         if (ksess == NULL) {
1164                 DBG3("No kernel session when tearing down session");
1165                 return;
1166         }
1167
1168         DBG("Tearing down kernel session");
1169
1170         /*
1171          * Destroy channels on the consumer if at least one FD has been sent and we
1172          * are in no output mode because the streams are in *no* monitor mode so we
1173          * have to send a command to clean them up or else they leaked.
1174          */
1175         if (!ksess->output_traces && ksess->consumer_fds_sent) {
1176                 int ret;
1177                 struct consumer_socket *socket;
1178                 struct lttng_ht_iter iter;
1179
1180                 /* For each consumer socket. */
1181                 rcu_read_lock();
1182                 cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
1183                                 socket, node.node) {
1184                         struct ltt_kernel_channel *chan;
1185
1186                         /* For each channel, ask the consumer to destroy it. */
1187                         cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1188                                 ret = kernel_consumer_destroy_channel(socket, chan);
1189                                 if (ret < 0) {
1190                                         /* Consumer is probably dead. Use next socket. */
1191                                         continue;
1192                                 }
1193                         }
1194                 }
1195                 rcu_read_unlock();
1196         }
1197
1198         /* Close any relayd session */
1199         consumer_output_send_destroy_relayd(ksess->consumer);
1200
1201         trace_kernel_destroy_session(ksess);
1202 }
1203
1204 /*
1205  * Destroy a kernel channel object. It does not do anything on the tracer side.
1206  */
1207 void kernel_destroy_channel(struct ltt_kernel_channel *kchan)
1208 {
1209         struct ltt_kernel_session *ksess = NULL;
1210
1211         assert(kchan);
1212         assert(kchan->channel);
1213
1214         DBG3("Kernel destroy channel %s", kchan->channel->name);
1215
1216         /* Update channel count of associated session. */
1217         if (kchan->session) {
1218                 /* Keep pointer reference so we can update it after the destroy. */
1219                 ksess = kchan->session;
1220         }
1221
1222         trace_kernel_destroy_channel(kchan);
1223
1224         /*
1225          * At this point the kernel channel is not visible anymore. This is safe
1226          * since in order to work on a visible kernel session, the tracing session
1227          * lock (ltt_session.lock) MUST be acquired.
1228          */
1229         if (ksess) {
1230                 ksess->channel_count--;
1231         }
1232 }
1233
1234 /*
1235  * Take a snapshot for a given kernel session.
1236  *
1237  * Return 0 on success or else return a LTTNG_ERR code.
1238  */
1239 int kernel_snapshot_record(struct ltt_kernel_session *ksess,
1240                 struct snapshot_output *output, int wait,
1241                 uint64_t nb_packets_per_stream)
1242 {
1243         int err, ret, saved_metadata_fd;
1244         struct consumer_socket *socket;
1245         struct lttng_ht_iter iter;
1246         struct ltt_kernel_metadata *saved_metadata;
1247         struct ltt_session *session;
1248         uint64_t trace_archive_id;
1249
1250         assert(ksess);
1251         assert(ksess->consumer);
1252         assert(output);
1253
1254         DBG("Kernel snapshot record started");
1255
1256         session = session_find_by_id(ksess->id);
1257         assert(session);
1258         assert(pthread_mutex_trylock(&session->lock));
1259         assert(session_trylock_list());
1260         trace_archive_id = session->current_archive_id;
1261
1262         /* Save current metadata since the following calls will change it. */
1263         saved_metadata = ksess->metadata;
1264         saved_metadata_fd = ksess->metadata_stream_fd;
1265
1266         rcu_read_lock();
1267
1268         ret = kernel_open_metadata(ksess);
1269         if (ret < 0) {
1270                 ret = LTTNG_ERR_KERN_META_FAIL;
1271                 goto error;
1272         }
1273
1274         ret = kernel_open_metadata_stream(ksess);
1275         if (ret < 0) {
1276                 ret = LTTNG_ERR_KERN_META_FAIL;
1277                 goto error_open_stream;
1278         }
1279
1280         /* Send metadata to consumer and snapshot everything. */
1281         cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
1282                         socket, node.node) {
1283                 struct consumer_output *saved_output;
1284                 struct ltt_kernel_channel *chan;
1285
1286                 /*
1287                  * Temporarly switch consumer output for our snapshot output. As long
1288                  * as the session lock is taken, this is safe.
1289                  */
1290                 saved_output = ksess->consumer;
1291                 ksess->consumer = output->consumer;
1292
1293                 pthread_mutex_lock(socket->lock);
1294                 /* This stream must not be monitored by the consumer. */
1295                 ret = kernel_consumer_add_metadata(socket, ksess, 0);
1296                 pthread_mutex_unlock(socket->lock);
1297                 /* Put back the saved consumer output into the session. */
1298                 ksess->consumer = saved_output;
1299                 if (ret < 0) {
1300                         ret = LTTNG_ERR_KERN_CONSUMER_FAIL;
1301                         goto error_consumer;
1302                 }
1303
1304                 /* For each channel, ask the consumer to snapshot it. */
1305                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1306                         ret = consumer_snapshot_channel(socket, chan->key, output, 0,
1307                                         ksess->uid, ksess->gid,
1308                                         DEFAULT_KERNEL_TRACE_DIR, wait,
1309                                         nb_packets_per_stream,
1310                                         trace_archive_id);
1311                         if (ret < 0) {
1312                                 ret = LTTNG_ERR_KERN_CONSUMER_FAIL;
1313                                 (void) kernel_consumer_destroy_metadata(socket,
1314                                                 ksess->metadata);
1315                                 goto error_consumer;
1316                         }
1317                 }
1318
1319                 /* Snapshot metadata, */
1320                 ret = consumer_snapshot_channel(socket, ksess->metadata->key, output,
1321                                 1, ksess->uid, ksess->gid,
1322                                 DEFAULT_KERNEL_TRACE_DIR, wait, 0,
1323                                 trace_archive_id);
1324                 if (ret < 0) {
1325                         ret = LTTNG_ERR_KERN_CONSUMER_FAIL;
1326                         goto error_consumer;
1327                 }
1328
1329                 /*
1330                  * The metadata snapshot is done, ask the consumer to destroy it since
1331                  * it's not monitored on the consumer side.
1332                  */
1333                 (void) kernel_consumer_destroy_metadata(socket, ksess->metadata);
1334         }
1335
1336         ret = LTTNG_OK;
1337
1338 error_consumer:
1339         /* Close newly opened metadata stream. It's now on the consumer side. */
1340         err = close(ksess->metadata_stream_fd);
1341         if (err < 0) {
1342                 PERROR("close snapshot kernel");
1343         }
1344
1345 error_open_stream:
1346         trace_kernel_destroy_metadata(ksess->metadata);
1347 error:
1348         /* Restore metadata state.*/
1349         ksess->metadata = saved_metadata;
1350         ksess->metadata_stream_fd = saved_metadata_fd;
1351
1352         rcu_read_unlock();
1353         return ret;
1354 }
1355
1356 /*
1357  * Get the syscall mask array from the kernel tracer.
1358  *
1359  * Return 0 on success else a negative value. In both case, syscall_mask should
1360  * be freed.
1361  */
1362 int kernel_syscall_mask(int chan_fd, char **syscall_mask, uint32_t *nr_bits)
1363 {
1364         assert(syscall_mask);
1365         assert(nr_bits);
1366
1367         return kernctl_syscall_mask(chan_fd, syscall_mask, nr_bits);
1368 }
1369
1370 /*
1371  * Check for the support of the RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS via abi
1372  * version number.
1373  *
1374  * Return 1 on success, 0 when feature is not supported, negative value in case
1375  * of errors.
1376  */
1377 int kernel_supports_ring_buffer_snapshot_sample_positions(int tracer_fd)
1378 {
1379         int ret = 0; // Not supported by default
1380         struct lttng_kernel_tracer_abi_version abi;
1381
1382         ret = kernctl_tracer_abi_version(tracer_fd, &abi);
1383         if (ret < 0) {
1384                 ERR("Failed to retrieve lttng-modules ABI version");
1385                 goto error;
1386         }
1387
1388         /*
1389          * RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS was introduced in 2.3
1390          */
1391         if (abi.major >= 2 && abi.minor >= 3) {
1392                 /* Supported */
1393                 ret = 1;
1394         } else {
1395                 /* Not supported */
1396                 ret = 0;
1397         }
1398 error:
1399         return ret;
1400 }
1401
1402 /*
1403  * Rotate a kernel session.
1404  *
1405  * Return 0 on success or else return a LTTNG_ERR code.
1406  */
1407 int kernel_rotate_session(struct ltt_session *session)
1408 {
1409         int ret;
1410         struct consumer_socket *socket;
1411         struct lttng_ht_iter iter;
1412         struct ltt_kernel_session *ksess = session->kernel_session;
1413
1414         assert(ksess);
1415         assert(ksess->consumer);
1416
1417         DBG("Rotate kernel session %s started (session %" PRIu64 ")",
1418                         session->name, session->id);
1419
1420         rcu_read_lock();
1421
1422         /*
1423          * Note that this loop will end after one iteration given that there is
1424          * only one kernel consumer.
1425          */
1426         cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
1427                         socket, node.node) {
1428                 struct ltt_kernel_channel *chan;
1429
1430                 /*
1431                  * Account the metadata channel first to make sure the
1432                  * number of channels waiting for a rotation cannot
1433                  * reach 0 before we complete the iteration over all
1434                  * the channels.
1435                  */
1436                 ret = rotate_add_channel_pending(ksess->metadata->key,
1437                                 LTTNG_DOMAIN_KERNEL, session);
1438                 if (ret < 0) {
1439                         ret = LTTNG_ERR_KERN_CONSUMER_FAIL;
1440                         goto error;
1441                 }
1442
1443                 /* For each channel, ask the consumer to rotate it. */
1444                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1445                         ret = rotate_add_channel_pending(chan->key,
1446                                         LTTNG_DOMAIN_KERNEL, session);
1447                         if (ret < 0) {
1448                                 ret = LTTNG_ERR_KERN_CONSUMER_FAIL;
1449                                 goto error;
1450                         }
1451
1452                         DBG("Rotate channel %" PRIu64 ", session %s", chan->key, session->name);
1453                         ret = consumer_rotate_channel(socket, chan->key,
1454                                         ksess->uid, ksess->gid, ksess->consumer,
1455                                         ksess->consumer->subdir,
1456                                         /* is_metadata_channel */ false,
1457                                         session->current_archive_id,
1458                                         &session->rotate_pending_relay);
1459                         if (ret < 0) {
1460                                 ret = LTTNG_ERR_KERN_CONSUMER_FAIL;
1461                                 goto error;
1462                         }
1463                 }
1464
1465                 /*
1466                  * Rotate the metadata channel.
1467                  */
1468                 ret = consumer_rotate_channel(socket, ksess->metadata->key,
1469                                 ksess->uid, ksess->gid, ksess->consumer,
1470                                 ksess->consumer->subdir,
1471                                 /* is_metadata_channel */ true,
1472                                 session->current_archive_id,
1473                                 &session->rotate_pending_relay);
1474                 if (ret < 0) {
1475                         ret = LTTNG_ERR_KERN_CONSUMER_FAIL;
1476                         goto error;
1477                 }
1478         }
1479
1480         ret = LTTNG_OK;
1481
1482 error:
1483         rcu_read_unlock();
1484         return ret;
1485 }