Lines Matching +full:non +full:- +full:overlap +full:- +full:time
1 // SPDX-License-Identifier: GPL-2.0-only
4 * Copyright (c) 2013-2014, Intel Corporation.
22 #include "intel-pt-insn-decoder.h"
23 #include "intel-pt-pkt-decoder.h"
24 #include "intel-pt-decoder.h"
25 #include "intel-pt-log.h"
111 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
274 if (decoder->print_once & bit) in intel_pt_print_once()
276 decoder->print_once |= bit; in intel_pt_print_once()
290 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { in intel_pt_setup_period()
293 period = intel_pt_lower_power_of_2(decoder->period); in intel_pt_setup_period()
294 decoder->period_mask = ~(period - 1); in intel_pt_setup_period()
295 decoder->period_ticks = period; in intel_pt_setup_period()
310 if (!params->get_trace || !params->walk_insn) in intel_pt_decoder_new()
317 decoder->get_trace = params->get_trace; in intel_pt_decoder_new()
318 decoder->walk_insn = params->walk_insn; in intel_pt_decoder_new()
319 decoder->pgd_ip = params->pgd_ip; in intel_pt_decoder_new()
320 decoder->lookahead = params->lookahead; in intel_pt_decoder_new()
321 decoder->findnew_vmcs_info = params->findnew_vmcs_info; in intel_pt_decoder_new()
322 decoder->data = params->data; in intel_pt_decoder_new()
323 decoder->return_compression = params->return_compression; in intel_pt_decoder_new()
324 decoder->branch_enable = params->branch_enable; in intel_pt_decoder_new()
325 decoder->hop = params->quick >= 1; in intel_pt_decoder_new()
326 decoder->leap = params->quick >= 2; in intel_pt_decoder_new()
327 decoder->vm_time_correlation = params->vm_time_correlation; in intel_pt_decoder_new()
328 decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run; in intel_pt_decoder_new()
329 decoder->first_timestamp = params->first_timestamp; in intel_pt_decoder_new()
330 decoder->last_reliable_timestamp = params->first_timestamp; in intel_pt_decoder_new()
331 decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS; in intel_pt_decoder_new()
333 decoder->flags = params->flags; in intel_pt_decoder_new()
335 decoder->ctl = params->ctl; in intel_pt_decoder_new()
336 decoder->period = params->period; in intel_pt_decoder_new()
337 decoder->period_type = params->period_type; in intel_pt_decoder_new()
339 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; in intel_pt_decoder_new()
340 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; in intel_pt_decoder_new()
342 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); in intel_pt_decoder_new()
346 decoder->mtc_shift = params->mtc_period; in intel_pt_decoder_new()
347 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; in intel_pt_decoder_new()
349 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; in intel_pt_decoder_new()
350 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; in intel_pt_decoder_new()
352 if (!decoder->tsc_ctc_ratio_n) in intel_pt_decoder_new()
353 decoder->tsc_ctc_ratio_d = 0; in intel_pt_decoder_new()
355 if (decoder->tsc_ctc_ratio_d) { in intel_pt_decoder_new()
356 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) in intel_pt_decoder_new()
357 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / in intel_pt_decoder_new()
358 decoder->tsc_ctc_ratio_d; in intel_pt_decoder_new()
367 decoder->tsc_slip = 0x10000; in intel_pt_decoder_new()
369 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); in intel_pt_decoder_new()
370 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); in intel_pt_decoder_new()
371 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); in intel_pt_decoder_new()
372 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); in intel_pt_decoder_new()
373 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); in intel_pt_decoder_new()
375 if (decoder->hop) in intel_pt_decoder_new()
384 decoder->first_timestamp = first_timestamp; in intel_pt_set_first_timestamp()
389 struct intel_pt_blk *blk = stack->blk; in intel_pt_pop_blk()
391 stack->blk = blk->prev; in intel_pt_pop_blk()
392 if (!stack->spare) in intel_pt_pop_blk()
393 stack->spare = blk; in intel_pt_pop_blk()
400 if (!stack->pos) { in intel_pt_pop()
401 if (!stack->blk) in intel_pt_pop()
404 if (!stack->blk) in intel_pt_pop()
406 stack->pos = INTEL_PT_BLK_SIZE; in intel_pt_pop()
408 return stack->blk->ip[--stack->pos]; in intel_pt_pop()
415 if (stack->spare) { in intel_pt_alloc_blk()
416 blk = stack->spare; in intel_pt_alloc_blk()
417 stack->spare = NULL; in intel_pt_alloc_blk()
421 return -ENOMEM; in intel_pt_alloc_blk()
424 blk->prev = stack->blk; in intel_pt_alloc_blk()
425 stack->blk = blk; in intel_pt_alloc_blk()
426 stack->pos = 0; in intel_pt_alloc_blk()
434 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { in intel_pt_push()
440 stack->blk->ip[stack->pos++] = ip; in intel_pt_push()
446 while (stack->blk) in intel_pt_clear_stack()
448 stack->pos = 0; in intel_pt_clear_stack()
454 zfree(&stack->blk); in intel_pt_free_stack()
455 zfree(&stack->spare); in intel_pt_free_stack()
460 intel_pt_free_stack(&decoder->stack); in intel_pt_decoder_free()
467 case -ENOMEM: in intel_pt_ext_err()
469 case -ENOSYS: in intel_pt_ext_err()
471 case -EBADMSG: in intel_pt_ext_err()
473 case -ENODATA: in intel_pt_ext_err()
475 case -EILSEQ: in intel_pt_ext_err()
477 case -ENOENT: in intel_pt_ext_err()
479 case -EOVERFLOW: in intel_pt_ext_err()
481 case -ENOSPC: in intel_pt_ext_err()
483 case -ELOOP: in intel_pt_ext_err()
485 case -ECONNRESET: in intel_pt_ext_err()
502 [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",
519 switch (packet->count) { in intel_pt_calc_ip()
522 packet->payload; in intel_pt_calc_ip()
526 packet->payload; in intel_pt_calc_ip()
529 ip = packet->payload; in intel_pt_calc_ip()
530 /* Sign-extend 6-byte ip */ in intel_pt_calc_ip()
536 packet->payload; in intel_pt_calc_ip()
539 ip = packet->payload; in intel_pt_calc_ip()
550 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); in intel_pt_set_last_ip()
551 decoder->have_last_ip = true; in intel_pt_set_last_ip()
557 decoder->ip = decoder->last_ip; in intel_pt_set_ip()
562 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, in intel_pt_decoder_log_packet()
563 decoder->buf); in intel_pt_decoder_log_packet()
569 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; in intel_pt_bug()
570 return -ENOSYS; in intel_pt_bug()
575 decoder->tx_flags = 0; in intel_pt_clear_tx_flags()
580 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; in intel_pt_update_in_tx()
585 decoder->pip_payload = decoder->packet.payload; in intel_pt_update_pip()
590 decoder->next_nr = decoder->pip_payload & 1; in intel_pt_update_nr()
595 decoder->nr = decoder->pip_payload & 1; in intel_pt_set_nr()
596 decoder->next_nr = decoder->nr; in intel_pt_set_nr()
608 decoder->have_tma = false; in intel_pt_bad_packet()
609 decoder->pkt_len = 1; in intel_pt_bad_packet()
610 decoder->pkt_step = 1; in intel_pt_bad_packet()
612 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { in intel_pt_bad_packet()
614 decoder->pkt_state = INTEL_PT_STATE_ERR1; in intel_pt_bad_packet()
616 return -EBADMSG; in intel_pt_bad_packet()
621 decoder->sample_timestamp = decoder->timestamp; in intel_pt_update_sample_time()
622 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; in intel_pt_update_sample_time()
623 decoder->state.cycles = decoder->tot_cyc_cnt; in intel_pt_update_sample_time()
628 decoder->ip = 0; in intel_pt_reposition()
629 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; in intel_pt_reposition()
630 decoder->timestamp = 0; in intel_pt_reposition()
631 decoder->have_tma = false; in intel_pt_reposition()
639 decoder->pkt_step = 0; in intel_pt_get_data()
642 ret = decoder->get_trace(&buffer, decoder->data); in intel_pt_get_data()
645 decoder->buf = buffer.buf; in intel_pt_get_data()
646 decoder->len = buffer.len; in intel_pt_get_data()
647 if (!decoder->len) { in intel_pt_get_data()
649 return -ENODATA; in intel_pt_get_data()
651 decoder->buf_timestamp = buffer.ref_timestamp; in intel_pt_get_data()
654 decoder->ref_timestamp = buffer.ref_timestamp; in intel_pt_get_data()
655 decoder->state.trace_nr = buffer.trace_nr; in intel_pt_get_data()
656 decoder->vm_tm_corr_same_buf = false; in intel_pt_get_data()
658 decoder->ref_timestamp); in intel_pt_get_data()
659 return -ENOLINK; in intel_pt_get_data()
668 if (!decoder->next_buf) in intel_pt_get_next_data()
671 decoder->buf = decoder->next_buf; in intel_pt_get_next_data()
672 decoder->len = decoder->next_len; in intel_pt_get_next_data()
673 decoder->next_buf = 0; in intel_pt_get_next_data()
674 decoder->next_len = 0; in intel_pt_get_next_data()
680 unsigned char *buf = decoder->temp_buf; in intel_pt_get_split_packet()
684 old_len = decoder->len; in intel_pt_get_split_packet()
685 len = decoder->len; in intel_pt_get_split_packet()
686 memcpy(buf, decoder->buf, len); in intel_pt_get_split_packet()
690 decoder->pos += old_len; in intel_pt_get_split_packet()
691 return ret < 0 ? ret : -EINVAL; in intel_pt_get_split_packet()
694 n = INTEL_PT_PKT_MAX_SZ - len; in intel_pt_get_split_packet()
695 if (n > decoder->len) in intel_pt_get_split_packet()
696 n = decoder->len; in intel_pt_get_split_packet()
697 memcpy(buf + len, decoder->buf, n); in intel_pt_get_split_packet()
700 decoder->prev_pkt_ctx = decoder->pkt_ctx; in intel_pt_get_split_packet()
701 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); in intel_pt_get_split_packet()
703 decoder->next_buf = decoder->buf; in intel_pt_get_split_packet()
704 decoder->next_len = decoder->len; in intel_pt_get_split_packet()
705 decoder->buf = buf; in intel_pt_get_split_packet()
706 decoder->len = old_len; in intel_pt_get_split_packet()
710 decoder->next_buf = decoder->buf + (ret - old_len); in intel_pt_get_split_packet()
711 decoder->next_len = decoder->len - (ret - old_len); in intel_pt_get_split_packet()
713 decoder->buf = buf; in intel_pt_get_split_packet()
714 decoder->len = ret; in intel_pt_get_split_packet()
735 const unsigned char *buf = decoder->buf; in intel_pt_pkt_lookahead()
736 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; in intel_pt_pkt_lookahead()
737 size_t len = decoder->len; in intel_pt_pkt_lookahead()
741 pkt_info.pos = decoder->pos; in intel_pt_pkt_lookahead()
742 pkt_info.pkt_len = decoder->pkt_step; in intel_pt_pkt_lookahead()
743 pkt_info.last_packet_type = decoder->last_packet_type; in intel_pt_pkt_lookahead()
750 len -= pkt_info.pkt_len; in intel_pt_pkt_lookahead()
788 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
790 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
797 uint32_t first_missing_bit = 1U << (16 - mtc_shift); in intel_pt_fixup_last_mtc()
798 uint32_t mask = ~(first_missing_bit - 1); in intel_pt_fixup_last_mtc()
802 *last_mtc -= first_missing_bit; in intel_pt_fixup_last_mtc()
809 struct intel_pt_decoder *decoder = pkt_info->decoder; in intel_pt_calc_cyc_cb()
810 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; in intel_pt_calc_cyc_cb()
816 switch (pkt_info->packet.type) { in intel_pt_calc_cyc_cb()
841 if (!data->have_tma) in intel_pt_calc_cyc_cb()
844 mtc = pkt_info->packet.payload; in intel_pt_calc_cyc_cb()
845 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { in intel_pt_calc_cyc_cb()
846 data->fixup_last_mtc = false; in intel_pt_calc_cyc_cb()
847 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, in intel_pt_calc_cyc_cb()
848 &data->last_mtc); in intel_pt_calc_cyc_cb()
850 if (mtc > data->last_mtc) in intel_pt_calc_cyc_cb()
851 mtc_delta = mtc - data->last_mtc; in intel_pt_calc_cyc_cb()
853 mtc_delta = mtc + 256 - data->last_mtc; in intel_pt_calc_cyc_cb()
854 data->ctc_delta += mtc_delta << decoder->mtc_shift; in intel_pt_calc_cyc_cb()
855 data->last_mtc = mtc; in intel_pt_calc_cyc_cb()
857 if (decoder->tsc_ctc_mult) { in intel_pt_calc_cyc_cb()
858 timestamp = data->ctc_timestamp + in intel_pt_calc_cyc_cb()
859 data->ctc_delta * decoder->tsc_ctc_mult; in intel_pt_calc_cyc_cb()
861 timestamp = data->ctc_timestamp + in intel_pt_calc_cyc_cb()
862 multdiv(data->ctc_delta, in intel_pt_calc_cyc_cb()
863 decoder->tsc_ctc_ratio_n, in intel_pt_calc_cyc_cb()
864 decoder->tsc_ctc_ratio_d); in intel_pt_calc_cyc_cb()
867 if (timestamp < data->timestamp) in intel_pt_calc_cyc_cb()
870 if (pkt_info->last_packet_type != INTEL_PT_CYC) { in intel_pt_calc_cyc_cb()
871 data->timestamp = timestamp; in intel_pt_calc_cyc_cb()
879 * For now, do not support using TSC packets - refer in intel_pt_calc_cyc_cb()
882 if (data->from_mtc) in intel_pt_calc_cyc_cb()
884 timestamp = pkt_info->packet.payload | in intel_pt_calc_cyc_cb()
885 (data->timestamp & (0xffULL << 56)); in intel_pt_calc_cyc_cb()
886 if (data->from_mtc && timestamp < data->timestamp && in intel_pt_calc_cyc_cb()
887 data->timestamp - timestamp < decoder->tsc_slip) in intel_pt_calc_cyc_cb()
889 if (timestamp < data->timestamp) in intel_pt_calc_cyc_cb()
891 if (pkt_info->last_packet_type != INTEL_PT_CYC) { in intel_pt_calc_cyc_cb()
892 if (data->from_mtc) in intel_pt_calc_cyc_cb()
894 data->tsc_timestamp = timestamp; in intel_pt_calc_cyc_cb()
895 data->timestamp = timestamp; in intel_pt_calc_cyc_cb()
901 if (data->from_mtc) in intel_pt_calc_cyc_cb()
904 if (!decoder->tsc_ctc_ratio_d) in intel_pt_calc_cyc_cb()
907 ctc = pkt_info->packet.payload; in intel_pt_calc_cyc_cb()
908 fc = pkt_info->packet.count; in intel_pt_calc_cyc_cb()
909 ctc_rem = ctc & decoder->ctc_rem_mask; in intel_pt_calc_cyc_cb()
911 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; in intel_pt_calc_cyc_cb()
913 data->ctc_timestamp = data->tsc_timestamp - fc; in intel_pt_calc_cyc_cb()
914 if (decoder->tsc_ctc_mult) { in intel_pt_calc_cyc_cb()
915 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; in intel_pt_calc_cyc_cb()
917 data->ctc_timestamp -= in intel_pt_calc_cyc_cb()
918 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, in intel_pt_calc_cyc_cb()
919 decoder->tsc_ctc_ratio_d); in intel_pt_calc_cyc_cb()
922 data->ctc_delta = 0; in intel_pt_calc_cyc_cb()
923 data->have_tma = true; in intel_pt_calc_cyc_cb()
924 data->fixup_last_mtc = true; in intel_pt_calc_cyc_cb()
929 data->cycle_cnt += pkt_info->packet.payload; in intel_pt_calc_cyc_cb()
933 cbr = pkt_info->packet.payload; in intel_pt_calc_cyc_cb()
934 if (data->cbr && data->cbr != cbr) in intel_pt_calc_cyc_cb()
936 data->cbr = cbr; in intel_pt_calc_cyc_cb()
937 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; in intel_pt_calc_cyc_cb()
953 if (!data->cbr && decoder->cbr) { in intel_pt_calc_cyc_cb()
954 data->cbr = decoder->cbr; in intel_pt_calc_cyc_cb()
955 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; in intel_pt_calc_cyc_cb()
958 if (!data->cycle_cnt) in intel_pt_calc_cyc_cb()
961 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; in intel_pt_calc_cyc_cb()
963 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && in intel_pt_calc_cyc_cb()
964 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { in intel_pt_calc_cyc_cb()
965 …intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos … in intel_pt_calc_cyc_cb()
966 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); in intel_pt_calc_cyc_cb()
970 decoder->calc_cyc_to_tsc = cyc_to_tsc; in intel_pt_calc_cyc_cb()
971 decoder->have_calc_cyc_to_tsc = true; in intel_pt_calc_cyc_cb()
973 if (data->cbr) { in intel_pt_calc_cyc_cb()
974 …intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt … in intel_pt_calc_cyc_cb()
975 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); in intel_pt_calc_cyc_cb()
977 …intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64… in intel_pt_calc_cyc_cb()
978 cyc_to_tsc, pkt_info->pos); in intel_pt_calc_cyc_cb()
990 .last_mtc = decoder->last_mtc, in intel_pt_calc_cyc_to_tsc()
991 .ctc_timestamp = decoder->ctc_timestamp, in intel_pt_calc_cyc_to_tsc()
992 .ctc_delta = decoder->ctc_delta, in intel_pt_calc_cyc_to_tsc()
993 .tsc_timestamp = decoder->tsc_timestamp, in intel_pt_calc_cyc_to_tsc()
994 .timestamp = decoder->timestamp, in intel_pt_calc_cyc_to_tsc()
995 .have_tma = decoder->have_tma, in intel_pt_calc_cyc_to_tsc()
996 .fixup_last_mtc = decoder->fixup_last_mtc, in intel_pt_calc_cyc_to_tsc()
1016 decoder->last_packet_type = decoder->packet.type; in intel_pt_get_next_packet()
1019 decoder->pos += decoder->pkt_step; in intel_pt_get_next_packet()
1020 decoder->buf += decoder->pkt_step; in intel_pt_get_next_packet()
1021 decoder->len -= decoder->pkt_step; in intel_pt_get_next_packet()
1023 if (!decoder->len) { in intel_pt_get_next_packet()
1029 decoder->prev_pkt_ctx = decoder->pkt_ctx; in intel_pt_get_next_packet()
1030 ret = intel_pt_get_packet(decoder->buf, decoder->len, in intel_pt_get_next_packet()
1031 &decoder->packet, &decoder->pkt_ctx); in intel_pt_get_next_packet()
1033 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { in intel_pt_get_next_packet()
1041 decoder->pkt_len = ret; in intel_pt_get_next_packet()
1042 decoder->pkt_step = ret; in intel_pt_get_next_packet()
1044 } while (decoder->packet.type == INTEL_PT_PAD); in intel_pt_get_next_packet()
1053 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; in intel_pt_next_period()
1054 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_next_period()
1055 if (decoder->continuous_period) { in intel_pt_next_period()
1056 if (masked_timestamp > decoder->last_masked_timestamp) in intel_pt_next_period()
1060 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_next_period()
1061 if (masked_timestamp > decoder->last_masked_timestamp) { in intel_pt_next_period()
1062 decoder->last_masked_timestamp = masked_timestamp; in intel_pt_next_period()
1063 decoder->continuous_period = true; in intel_pt_next_period()
1067 if (masked_timestamp < decoder->last_masked_timestamp) in intel_pt_next_period()
1068 return decoder->period_ticks; in intel_pt_next_period()
1070 return decoder->period_ticks - (timestamp - masked_timestamp); in intel_pt_next_period()
1075 switch (decoder->period_type) { in intel_pt_next_sample()
1077 return decoder->period - decoder->period_insn_cnt; in intel_pt_next_sample()
1091 switch (decoder->period_type) { in intel_pt_sample_insn()
1093 decoder->period_insn_cnt = 0; in intel_pt_sample_insn()
1096 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; in intel_pt_sample_insn()
1097 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_sample_insn()
1098 if (masked_timestamp > decoder->last_masked_timestamp) in intel_pt_sample_insn()
1099 decoder->last_masked_timestamp = masked_timestamp; in intel_pt_sample_insn()
1101 decoder->last_masked_timestamp += decoder->period_ticks; in intel_pt_sample_insn()
1109 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_sample_insn()
1113 * Sample FUP instruction at the same time as reporting the FUP event, so the
1122 decoder->state.insn_op = INTEL_PT_OP_OTHER; in intel_pt_sample_fup_insn()
1123 decoder->state.insn_len = 0; in intel_pt_sample_fup_insn()
1125 if (!decoder->branch_enable || !decoder->pge || decoder->hop || in intel_pt_sample_fup_insn()
1126 decoder->ip != decoder->last_ip) in intel_pt_sample_fup_insn()
1129 if (!decoder->mtc_insn) in intel_pt_sample_fup_insn()
1130 decoder->mtc_insn = true; in intel_pt_sample_fup_insn()
1136 err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip, in intel_pt_sample_fup_insn()
1137 0, max_insn_cnt, decoder->data); in intel_pt_sample_fup_insn()
1143 intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip); in intel_pt_sample_fup_insn()
1147 decoder->tot_insn_cnt += insn_cnt; in intel_pt_sample_fup_insn()
1148 decoder->timestamp_insn_cnt += insn_cnt; in intel_pt_sample_fup_insn()
1149 decoder->sample_insn_cnt += insn_cnt; in intel_pt_sample_fup_insn()
1150 decoder->period_insn_cnt += insn_cnt; in intel_pt_sample_fup_insn()
1154 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_sample_fup_insn()
1155 decoder->ip += intel_pt_insn.length; in intel_pt_sample_fup_insn()
1164 if (!decoder->mtc_insn) in intel_pt_walk_insn()
1165 decoder->mtc_insn = true; in intel_pt_walk_insn()
1169 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, in intel_pt_walk_insn()
1170 max_insn_cnt, decoder->data); in intel_pt_walk_insn()
1172 decoder->tot_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1173 decoder->timestamp_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1174 decoder->sample_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1175 decoder->period_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1178 decoder->no_progress = 0; in intel_pt_walk_insn()
1179 decoder->pkt_state = INTEL_PT_STATE_ERR2; in intel_pt_walk_insn()
1181 decoder->ip); in intel_pt_walk_insn()
1182 if (err == -ENOENT) in intel_pt_walk_insn()
1183 return -ENOLINK; in intel_pt_walk_insn()
1184 return -EILSEQ; in intel_pt_walk_insn()
1187 if (ip && decoder->ip == ip) { in intel_pt_walk_insn()
1188 err = -EAGAIN; in intel_pt_walk_insn()
1195 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { in intel_pt_walk_insn()
1196 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_insn()
1197 decoder->state.from_ip = decoder->ip; in intel_pt_walk_insn()
1198 decoder->state.to_ip = 0; in intel_pt_walk_insn()
1199 decoder->ip += intel_pt_insn->length; in intel_pt_walk_insn()
1204 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { in intel_pt_walk_insn()
1205 /* Zero-length calls are excluded */ in intel_pt_walk_insn()
1206 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || in intel_pt_walk_insn()
1207 intel_pt_insn->rel) { in intel_pt_walk_insn()
1208 err = intel_pt_push(&decoder->stack, decoder->ip + in intel_pt_walk_insn()
1209 intel_pt_insn->length); in intel_pt_walk_insn()
1213 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { in intel_pt_walk_insn()
1214 decoder->ret_addr = intel_pt_pop(&decoder->stack); in intel_pt_walk_insn()
1217 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { in intel_pt_walk_insn()
1218 int cnt = decoder->no_progress++; in intel_pt_walk_insn()
1220 decoder->state.from_ip = decoder->ip; in intel_pt_walk_insn()
1221 decoder->ip += intel_pt_insn->length + in intel_pt_walk_insn()
1222 intel_pt_insn->rel; in intel_pt_walk_insn()
1223 decoder->state.to_ip = decoder->ip; in intel_pt_walk_insn()
1235 decoder->stuck_ip = decoder->state.to_ip; in intel_pt_walk_insn()
1236 decoder->stuck_ip_prd = 1; in intel_pt_walk_insn()
1237 decoder->stuck_ip_cnt = 1; in intel_pt_walk_insn()
1238 } else if (cnt > decoder->max_loops || in intel_pt_walk_insn()
1239 decoder->state.to_ip == decoder->stuck_ip) { in intel_pt_walk_insn()
1240 intel_pt_log_at("ERROR: Never-ending loop", in intel_pt_walk_insn()
1241 decoder->state.to_ip); in intel_pt_walk_insn()
1242 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_insn()
1243 err = -ELOOP; in intel_pt_walk_insn()
1245 } else if (!--decoder->stuck_ip_cnt) { in intel_pt_walk_insn()
1246 decoder->stuck_ip_prd += 1; in intel_pt_walk_insn()
1247 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; in intel_pt_walk_insn()
1248 decoder->stuck_ip = decoder->state.to_ip; in intel_pt_walk_insn()
1254 decoder->no_progress = 0; in intel_pt_walk_insn()
1256 decoder->state.insn_op = intel_pt_insn->op; in intel_pt_walk_insn()
1257 decoder->state.insn_len = intel_pt_insn->length; in intel_pt_walk_insn()
1258 memcpy(decoder->state.insn, intel_pt_insn->buf, in intel_pt_walk_insn()
1261 if (decoder->tx_flags & INTEL_PT_IN_TX) in intel_pt_walk_insn()
1262 decoder->state.flags |= INTEL_PT_IN_TX; in intel_pt_walk_insn()
1269 bool iflag = decoder->packet.count & INTEL_PT_IFLAG; in intel_pt_mode_exec_status()
1271 decoder->exec_mode = decoder->packet.payload; in intel_pt_mode_exec_status()
1272 decoder->iflag = iflag; in intel_pt_mode_exec_status()
1273 decoder->next_iflag = iflag; in intel_pt_mode_exec_status()
1274 decoder->state.from_iflag = iflag; in intel_pt_mode_exec_status()
1275 decoder->state.to_iflag = iflag; in intel_pt_mode_exec_status()
1280 bool iflag = decoder->packet.count & INTEL_PT_IFLAG; in intel_pt_mode_exec()
1282 decoder->exec_mode = decoder->packet.payload; in intel_pt_mode_exec()
1283 decoder->next_iflag = iflag; in intel_pt_mode_exec()
1288 decoder->state.type |= INTEL_PT_IFLAG_CHG; in intel_pt_sample_iflag()
1289 decoder->state.from_iflag = decoder->iflag; in intel_pt_sample_iflag()
1290 decoder->state.to_iflag = decoder->next_iflag; in intel_pt_sample_iflag()
1291 decoder->iflag = decoder->next_iflag; in intel_pt_sample_iflag()
1296 if (decoder->iflag != decoder->next_iflag) in intel_pt_sample_iflag_chg()
1302 decoder->set_fup_tx_flags = false; in intel_pt_clear_fup_event()
1303 decoder->set_fup_ptw = false; in intel_pt_clear_fup_event()
1304 decoder->set_fup_mwait = false; in intel_pt_clear_fup_event()
1305 decoder->set_fup_pwre = false; in intel_pt_clear_fup_event()
1306 decoder->set_fup_exstop = false; in intel_pt_clear_fup_event()
1307 decoder->set_fup_bep = false; in intel_pt_clear_fup_event()
1308 decoder->set_fup_cfe_ip = false; in intel_pt_clear_fup_event()
1309 decoder->set_fup_cfe = false; in intel_pt_clear_fup_event()
1310 decoder->evd_cnt = 0; in intel_pt_clear_fup_event()
1311 decoder->set_fup_mode_exec = false; in intel_pt_clear_fup_event()
1312 decoder->iflag = decoder->next_iflag; in intel_pt_clear_fup_event()
1317 enum intel_pt_sample_type type = decoder->state.type; in intel_pt_fup_event()
1321 decoder->state.type &= ~INTEL_PT_BRANCH; in intel_pt_fup_event()
1323 if (decoder->set_fup_cfe_ip || decoder->set_fup_cfe) { in intel_pt_fup_event()
1324 bool ip = decoder->set_fup_cfe_ip; in intel_pt_fup_event()
1326 decoder->set_fup_cfe_ip = false; in intel_pt_fup_event()
1327 decoder->set_fup_cfe = false; in intel_pt_fup_event()
1328 decoder->state.type |= INTEL_PT_EVT; in intel_pt_fup_event()
1329 if (!ip && decoder->pge) in intel_pt_fup_event()
1330 decoder->state.type |= INTEL_PT_BRANCH; in intel_pt_fup_event()
1331 decoder->state.cfe_type = decoder->fup_cfe_pkt.count; in intel_pt_fup_event()
1332 decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload; in intel_pt_fup_event()
1333 decoder->state.evd_cnt = decoder->evd_cnt; in intel_pt_fup_event()
1334 decoder->state.evd = decoder->evd; in intel_pt_fup_event()
1335 decoder->evd_cnt = 0; in intel_pt_fup_event()
1336 if (ip || decoder->pge) in intel_pt_fup_event()
1337 decoder->state.flags |= INTEL_PT_FUP_IP; in intel_pt_fup_event()
1340 if (decoder->set_fup_mode_exec) { in intel_pt_fup_event()
1341 decoder->set_fup_mode_exec = false; in intel_pt_fup_event()
1346 if (decoder->set_fup_tx_flags) { in intel_pt_fup_event()
1347 decoder->set_fup_tx_flags = false; in intel_pt_fup_event()
1348 decoder->tx_flags = decoder->fup_tx_flags; in intel_pt_fup_event()
1349 decoder->state.type |= INTEL_PT_TRANSACTION; in intel_pt_fup_event()
1350 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) in intel_pt_fup_event()
1351 decoder->state.type |= INTEL_PT_BRANCH; in intel_pt_fup_event()
1352 decoder->state.flags = decoder->fup_tx_flags; in intel_pt_fup_event()
1355 if (decoder->set_fup_ptw) { in intel_pt_fup_event()
1356 decoder->set_fup_ptw = false; in intel_pt_fup_event()
1357 decoder->state.type |= INTEL_PT_PTW; in intel_pt_fup_event()
1358 decoder->state.flags |= INTEL_PT_FUP_IP; in intel_pt_fup_event()
1359 decoder->state.ptw_payload = decoder->fup_ptw_payload; in intel_pt_fup_event()
1362 if (decoder->set_fup_mwait) { in intel_pt_fup_event()
1363 decoder->set_fup_mwait = false; in intel_pt_fup_event()
1364 decoder->state.type |= INTEL_PT_MWAIT_OP; in intel_pt_fup_event()
1365 decoder->state.mwait_payload = decoder->fup_mwait_payload; in intel_pt_fup_event()
1368 if (decoder->set_fup_pwre) { in intel_pt_fup_event()
1369 decoder->set_fup_pwre = false; in intel_pt_fup_event()
1370 decoder->state.type |= INTEL_PT_PWR_ENTRY; in intel_pt_fup_event()
1371 decoder->state.pwre_payload = decoder->fup_pwre_payload; in intel_pt_fup_event()
1374 if (decoder->set_fup_exstop) { in intel_pt_fup_event()
1375 decoder->set_fup_exstop = false; in intel_pt_fup_event()
1376 decoder->state.type |= INTEL_PT_EX_STOP; in intel_pt_fup_event()
1377 decoder->state.flags |= INTEL_PT_FUP_IP; in intel_pt_fup_event()
1380 if (decoder->set_fup_bep) { in intel_pt_fup_event()
1381 decoder->set_fup_bep = false; in intel_pt_fup_event()
1382 decoder->state.type |= INTEL_PT_BLK_ITEMS; in intel_pt_fup_event()
1385 if (decoder->overflow) { in intel_pt_fup_event()
1386 decoder->overflow = false; in intel_pt_fup_event()
1387 if (!ret && !decoder->pge) { in intel_pt_fup_event()
1388 if (decoder->hop) { in intel_pt_fup_event()
1389 decoder->state.type = 0; in intel_pt_fup_event()
1390 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_fup_event()
1392 decoder->pge = true; in intel_pt_fup_event()
1393 decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN; in intel_pt_fup_event()
1394 decoder->state.from_ip = 0; in intel_pt_fup_event()
1395 decoder->state.to_ip = decoder->ip; in intel_pt_fup_event()
1400 decoder->state.from_ip = decoder->ip; in intel_pt_fup_event()
1401 decoder->state.to_ip = 0; in intel_pt_fup_event()
1405 decoder->state.type = type; in intel_pt_fup_event()
1414 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && in intel_pt_fup_with_nlip()
1415 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && in intel_pt_fup_with_nlip()
1416 ip == decoder->ip + intel_pt_insn->length; in intel_pt_fup_with_nlip()
1425 ip = decoder->last_ip; in intel_pt_walk_fup()
1431 if (err == -EAGAIN || in intel_pt_walk_fup()
1433 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; in intel_pt_walk_fup()
1435 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_fup()
1438 return -EAGAIN; in intel_pt_walk_fup()
1440 decoder->set_fup_tx_flags = false; in intel_pt_walk_fup()
1446 decoder->ip); in intel_pt_walk_fup()
1447 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_fup()
1448 return -ENOENT; in intel_pt_walk_fup()
1453 decoder->ip); in intel_pt_walk_fup()
1454 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_fup()
1455 return -ENOENT; in intel_pt_walk_fup()
1469 decoder->pgd_ip && in intel_pt_walk_tip()
1470 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && in intel_pt_walk_tip()
1471 (decoder->state.type & INTEL_PT_BRANCH) && in intel_pt_walk_tip()
1472 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { in intel_pt_walk_tip()
1474 decoder->no_progress = 0; in intel_pt_walk_tip()
1475 decoder->pge = false; in intel_pt_walk_tip()
1476 decoder->continuous_period = false; in intel_pt_walk_tip()
1477 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1478 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_tip()
1491 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { in intel_pt_walk_tip()
1492 decoder->pge = false; in intel_pt_walk_tip()
1493 decoder->continuous_period = false; in intel_pt_walk_tip()
1494 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1495 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1496 if (decoder->packet.count == 0) { in intel_pt_walk_tip()
1497 decoder->state.to_ip = 0; in intel_pt_walk_tip()
1499 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tip()
1500 decoder->ip = decoder->last_ip; in intel_pt_walk_tip()
1502 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_tip()
1504 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1505 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1506 if (decoder->packet.count == 0) { in intel_pt_walk_tip()
1507 decoder->state.to_ip = 0; in intel_pt_walk_tip()
1509 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tip()
1510 decoder->ip = decoder->last_ip; in intel_pt_walk_tip()
1517 uint64_t to_ip = decoder->ip + intel_pt_insn.length + in intel_pt_walk_tip()
1520 if (decoder->pgd_ip && in intel_pt_walk_tip()
1521 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && in intel_pt_walk_tip()
1522 decoder->pgd_ip(to_ip, decoder->data)) { in intel_pt_walk_tip()
1524 decoder->pge = false; in intel_pt_walk_tip()
1525 decoder->continuous_period = false; in intel_pt_walk_tip()
1526 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1527 decoder->ip = to_ip; in intel_pt_walk_tip()
1528 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1529 decoder->state.to_ip = to_ip; in intel_pt_walk_tip()
1530 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_tip()
1534 decoder->ip); in intel_pt_walk_tip()
1535 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_tip()
1536 return -ENOENT; in intel_pt_walk_tip()
1549 struct eptw_data *data = pkt_info->data; in intel_pt_eptw_lookahead_cb()
1552 switch (pkt_info->packet.type) { in intel_pt_eptw_lookahead_cb()
1584 nr_bits = data->bit_countdown; in intel_pt_eptw_lookahead_cb()
1585 if (nr_bits > pkt_info->packet.count) in intel_pt_eptw_lookahead_cb()
1586 nr_bits = pkt_info->packet.count; in intel_pt_eptw_lookahead_cb()
1587 data->payload <<= nr_bits; in intel_pt_eptw_lookahead_cb()
1588 data->payload |= pkt_info->packet.payload >> (64 - nr_bits); in intel_pt_eptw_lookahead_cb()
1589 data->bit_countdown -= nr_bits; in intel_pt_eptw_lookahead_cb()
1590 return !data->bit_countdown; in intel_pt_eptw_lookahead_cb()
1607 int n = 64 - decoder->tnt.count; in intel_pt_emulated_ptwrite()
1610 .payload = decoder->tnt.payload >> n, in intel_pt_emulated_ptwrite()
1613 decoder->emulated_ptwrite = false; in intel_pt_emulated_ptwrite()
1618 return -ECONNRESET; in intel_pt_emulated_ptwrite()
1620 decoder->state.type = INTEL_PT_PTW; in intel_pt_emulated_ptwrite()
1621 decoder->state.from_ip = decoder->ip; in intel_pt_emulated_ptwrite()
1622 decoder->state.to_ip = 0; in intel_pt_emulated_ptwrite()
1623 decoder->state.ptw_payload = data.payload; in intel_pt_emulated_ptwrite()
1633 if (decoder->emulated_ptwrite) in intel_pt_walk_tnt()
1637 decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite; in intel_pt_walk_tnt()
1641 decoder->emulated_ptwrite = false; in intel_pt_walk_tnt()
1646 if (!decoder->return_compression) { in intel_pt_walk_tnt()
1648 decoder->ip); in intel_pt_walk_tnt()
1649 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1650 return -ENOENT; in intel_pt_walk_tnt()
1652 if (!decoder->ret_addr) { in intel_pt_walk_tnt()
1654 decoder->ip); in intel_pt_walk_tnt()
1655 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1656 return -ENOENT; in intel_pt_walk_tnt()
1658 if (!(decoder->tnt.payload & BIT63)) { in intel_pt_walk_tnt()
1660 decoder->ip); in intel_pt_walk_tnt()
1661 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1662 return -ENOENT; in intel_pt_walk_tnt()
1664 decoder->tnt.count -= 1; in intel_pt_walk_tnt()
1665 if (decoder->tnt.count) in intel_pt_walk_tnt()
1666 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; in intel_pt_walk_tnt()
1668 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tnt()
1669 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1670 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1671 decoder->ip = decoder->ret_addr; in intel_pt_walk_tnt()
1672 decoder->state.to_ip = decoder->ip; in intel_pt_walk_tnt()
1681 if (decoder->packet.type != INTEL_PT_TIP || in intel_pt_walk_tnt()
1682 decoder->packet.count == 0) { in intel_pt_walk_tnt()
1684 decoder->ip); in intel_pt_walk_tnt()
1685 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1686 decoder->pkt_step = 0; in intel_pt_walk_tnt()
1687 return -ENOENT; in intel_pt_walk_tnt()
1690 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1691 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tnt()
1692 decoder->ip = decoder->last_ip; in intel_pt_walk_tnt()
1699 decoder->tnt.count -= 1; in intel_pt_walk_tnt()
1700 if (decoder->tnt.count) in intel_pt_walk_tnt()
1701 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; in intel_pt_walk_tnt()
1703 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tnt()
1704 if (decoder->tnt.payload & BIT63) { in intel_pt_walk_tnt()
1705 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1706 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1707 decoder->ip += intel_pt_insn.length + in intel_pt_walk_tnt()
1709 decoder->state.to_ip = decoder->ip; in intel_pt_walk_tnt()
1712 /* Instruction sample for a non-taken branch */ in intel_pt_walk_tnt()
1713 if (decoder->state.type & INTEL_PT_INSTRUCTION) { in intel_pt_walk_tnt()
1714 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1715 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_tnt()
1716 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1717 decoder->state.to_ip = 0; in intel_pt_walk_tnt()
1718 decoder->ip += intel_pt_insn.length; in intel_pt_walk_tnt()
1721 decoder->sample_cyc = false; in intel_pt_walk_tnt()
1722 decoder->ip += intel_pt_insn.length; in intel_pt_walk_tnt()
1723 if (!decoder->tnt.count) { in intel_pt_walk_tnt()
1725 return -EAGAIN; in intel_pt_walk_tnt()
1727 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1740 fup_tx_flags = decoder->packet.payload & in intel_pt_mode_tsx()
1745 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_mode_tsx()
1746 decoder->fup_tx_flags = fup_tx_flags; in intel_pt_mode_tsx()
1747 decoder->set_fup_tx_flags = true; in intel_pt_mode_tsx()
1748 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) in intel_pt_mode_tsx()
1752 decoder->pos); in intel_pt_mode_tsx()
1760 if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) { in intel_pt_evd()
1761 intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos); in intel_pt_evd()
1762 return -ENOSYS; in intel_pt_evd()
1764 decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){ in intel_pt_evd()
1765 .type = decoder->packet.count, in intel_pt_evd()
1766 .payload = decoder->packet.payload, in intel_pt_evd()
1776 if (ref_timestamp - timestamp > (1ULL << 55)) in intel_pt_8b_tsc()
1779 if (timestamp - ref_timestamp > (1ULL << 55)) in intel_pt_8b_tsc()
1780 timestamp -= (1ULL << 56); in intel_pt_8b_tsc()
1786 /* For use only when decoder->vm_time_correlation is true */
1790 uint64_t max_timestamp = decoder->buf_timestamp; in intel_pt_time_in_range()
1793 max_timestamp = decoder->last_reliable_timestamp + in intel_pt_time_in_range()
1796 return timestamp >= decoder->last_reliable_timestamp && in intel_pt_time_in_range()
1797 timestamp < decoder->buf_timestamp; in intel_pt_time_in_range()
1805 decoder->have_tma = false; in intel_pt_calc_tsc_timestamp()
1807 if (decoder->ref_timestamp) { in intel_pt_calc_tsc_timestamp()
1808 timestamp = intel_pt_8b_tsc(decoder->packet.payload, in intel_pt_calc_tsc_timestamp()
1809 decoder->ref_timestamp); in intel_pt_calc_tsc_timestamp()
1810 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1811 decoder->timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1812 decoder->ref_timestamp = 0; in intel_pt_calc_tsc_timestamp()
1813 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_tsc_timestamp()
1814 } else if (decoder->timestamp) { in intel_pt_calc_tsc_timestamp()
1815 timestamp = decoder->packet.payload | in intel_pt_calc_tsc_timestamp()
1816 (decoder->timestamp & (0xffULL << 56)); in intel_pt_calc_tsc_timestamp()
1817 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1818 if (timestamp < decoder->timestamp && in intel_pt_calc_tsc_timestamp()
1819 decoder->timestamp - timestamp < decoder->tsc_slip) { in intel_pt_calc_tsc_timestamp()
1822 timestamp = decoder->timestamp; in intel_pt_calc_tsc_timestamp()
1824 if (timestamp < decoder->timestamp) { in intel_pt_calc_tsc_timestamp()
1825 if (!decoder->buf_timestamp || in intel_pt_calc_tsc_timestamp()
1826 (timestamp + (1ULL << 56) < decoder->buf_timestamp)) { in intel_pt_calc_tsc_timestamp()
1829 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1832 timestamp = decoder->timestamp; in intel_pt_calc_tsc_timestamp()
1836 if (decoder->vm_time_correlation && in intel_pt_calc_tsc_timestamp()
1840 decoder->timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1841 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_tsc_timestamp()
1844 if (decoder->last_packet_type == INTEL_PT_CYC) { in intel_pt_calc_tsc_timestamp()
1845 decoder->cyc_ref_timestamp = decoder->timestamp; in intel_pt_calc_tsc_timestamp()
1846 decoder->cycle_cnt = 0; in intel_pt_calc_tsc_timestamp()
1847 decoder->have_calc_cyc_to_tsc = false; in intel_pt_calc_tsc_timestamp()
1851 intel_pt_log_to("Setting timestamp", decoder->timestamp); in intel_pt_calc_tsc_timestamp()
1859 decoder->timestamp_insn_cnt = 0; in intel_pt_overflow()
1860 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_overflow()
1861 decoder->state.from_ip = decoder->ip; in intel_pt_overflow()
1862 decoder->ip = 0; in intel_pt_overflow()
1863 decoder->pge = false; in intel_pt_overflow()
1865 decoder->overflow = true; in intel_pt_overflow()
1866 return -EOVERFLOW; in intel_pt_overflow()
1871 if (decoder->have_cyc) in intel_pt_mtc_cyc_cnt_pge()
1874 decoder->cyc_cnt_timestamp = decoder->timestamp; in intel_pt_mtc_cyc_cnt_pge()
1875 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; in intel_pt_mtc_cyc_cnt_pge()
1880 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; in intel_pt_mtc_cyc_cnt_cbr()
1882 if (decoder->pge) in intel_pt_mtc_cyc_cnt_cbr()
1890 if (decoder->have_cyc) in intel_pt_mtc_cyc_cnt_upd()
1893 decoder->sample_cyc = true; in intel_pt_mtc_cyc_cnt_upd()
1895 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) in intel_pt_mtc_cyc_cnt_upd()
1898 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; in intel_pt_mtc_cyc_cnt_upd()
1899 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; in intel_pt_mtc_cyc_cnt_upd()
1901 if (tot_cyc_cnt > decoder->tot_cyc_cnt) in intel_pt_mtc_cyc_cnt_upd()
1902 decoder->tot_cyc_cnt = tot_cyc_cnt; in intel_pt_mtc_cyc_cnt_upd()
1907 uint32_t ctc = decoder->packet.payload; in intel_pt_calc_tma()
1908 uint32_t fc = decoder->packet.count; in intel_pt_calc_tma()
1909 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; in intel_pt_calc_tma()
1911 if (!decoder->tsc_ctc_ratio_d) in intel_pt_calc_tma()
1914 if (decoder->pge && !decoder->in_psb) in intel_pt_calc_tma()
1919 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; in intel_pt_calc_tma()
1920 decoder->last_ctc = ctc - ctc_rem; in intel_pt_calc_tma()
1921 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; in intel_pt_calc_tma()
1922 if (decoder->tsc_ctc_mult) { in intel_pt_calc_tma()
1923 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; in intel_pt_calc_tma()
1925 decoder->ctc_timestamp -= multdiv(ctc_rem, in intel_pt_calc_tma()
1926 decoder->tsc_ctc_ratio_n, in intel_pt_calc_tma()
1927 decoder->tsc_ctc_ratio_d); in intel_pt_calc_tma()
1929 decoder->ctc_delta = 0; in intel_pt_calc_tma()
1930 decoder->have_tma = true; in intel_pt_calc_tma()
1931 decoder->fixup_last_mtc = true; in intel_pt_calc_tma()
1933 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); in intel_pt_calc_tma()
1941 if (!decoder->have_tma) in intel_pt_calc_mtc_timestamp()
1944 mtc = decoder->packet.payload; in intel_pt_calc_mtc_timestamp()
1946 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { in intel_pt_calc_mtc_timestamp()
1947 decoder->fixup_last_mtc = false; in intel_pt_calc_mtc_timestamp()
1948 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, in intel_pt_calc_mtc_timestamp()
1949 &decoder->last_mtc); in intel_pt_calc_mtc_timestamp()
1952 if (mtc > decoder->last_mtc) in intel_pt_calc_mtc_timestamp()
1953 mtc_delta = mtc - decoder->last_mtc; in intel_pt_calc_mtc_timestamp()
1955 mtc_delta = mtc + 256 - decoder->last_mtc; in intel_pt_calc_mtc_timestamp()
1957 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; in intel_pt_calc_mtc_timestamp()
1959 if (decoder->tsc_ctc_mult) { in intel_pt_calc_mtc_timestamp()
1960 timestamp = decoder->ctc_timestamp + in intel_pt_calc_mtc_timestamp()
1961 decoder->ctc_delta * decoder->tsc_ctc_mult; in intel_pt_calc_mtc_timestamp()
1963 timestamp = decoder->ctc_timestamp + in intel_pt_calc_mtc_timestamp()
1964 multdiv(decoder->ctc_delta, in intel_pt_calc_mtc_timestamp()
1965 decoder->tsc_ctc_ratio_n, in intel_pt_calc_mtc_timestamp()
1966 decoder->tsc_ctc_ratio_d); in intel_pt_calc_mtc_timestamp()
1969 if (timestamp < decoder->timestamp) in intel_pt_calc_mtc_timestamp()
1971 timestamp, decoder->timestamp); in intel_pt_calc_mtc_timestamp()
1973 decoder->timestamp = timestamp; in intel_pt_calc_mtc_timestamp()
1977 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_mtc_timestamp()
1978 decoder->last_mtc = mtc; in intel_pt_calc_mtc_timestamp()
1980 if (decoder->last_packet_type == INTEL_PT_CYC) { in intel_pt_calc_mtc_timestamp()
1981 decoder->cyc_ref_timestamp = decoder->timestamp; in intel_pt_calc_mtc_timestamp()
1982 decoder->cycle_cnt = 0; in intel_pt_calc_mtc_timestamp()
1983 decoder->have_calc_cyc_to_tsc = false; in intel_pt_calc_mtc_timestamp()
1987 intel_pt_log_to("Setting timestamp", decoder->timestamp); in intel_pt_calc_mtc_timestamp()
1992 unsigned int cbr = decoder->packet.payload & 0xff; in intel_pt_calc_cbr()
1994 decoder->cbr_payload = decoder->packet.payload; in intel_pt_calc_cbr()
1996 if (decoder->cbr == cbr) in intel_pt_calc_cbr()
1999 decoder->cbr = cbr; in intel_pt_calc_cbr()
2000 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; in intel_pt_calc_cbr()
2001 decoder->cyc_ref_timestamp = decoder->timestamp; in intel_pt_calc_cbr()
2002 decoder->cycle_cnt = 0; in intel_pt_calc_cbr()
2009 uint64_t timestamp = decoder->cyc_ref_timestamp; in intel_pt_calc_cyc_timestamp()
2011 decoder->have_cyc = true; in intel_pt_calc_cyc_timestamp()
2013 decoder->cycle_cnt += decoder->packet.payload; in intel_pt_calc_cyc_timestamp()
2014 if (decoder->pge) in intel_pt_calc_cyc_timestamp()
2015 decoder->tot_cyc_cnt += decoder->packet.payload; in intel_pt_calc_cyc_timestamp()
2016 decoder->sample_cyc = true; in intel_pt_calc_cyc_timestamp()
2018 if (!decoder->cyc_ref_timestamp) in intel_pt_calc_cyc_timestamp()
2021 if (decoder->have_calc_cyc_to_tsc) in intel_pt_calc_cyc_timestamp()
2022 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; in intel_pt_calc_cyc_timestamp()
2023 else if (decoder->cbr) in intel_pt_calc_cyc_timestamp()
2024 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; in intel_pt_calc_cyc_timestamp()
2028 if (timestamp < decoder->timestamp) in intel_pt_calc_cyc_timestamp()
2030 timestamp, decoder->timestamp); in intel_pt_calc_cyc_timestamp()
2032 decoder->timestamp = timestamp; in intel_pt_calc_cyc_timestamp()
2034 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_cyc_timestamp()
2036 intel_pt_log_to("Setting timestamp", decoder->timestamp); in intel_pt_calc_cyc_timestamp()
2041 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { in intel_pt_bbp()
2042 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); in intel_pt_bbp()
2043 decoder->state.items.is_32_bit = false; in intel_pt_bbp()
2045 decoder->blk_type = decoder->packet.payload; in intel_pt_bbp()
2046 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); in intel_pt_bbp()
2047 if (decoder->blk_type == INTEL_PT_GP_REGS) in intel_pt_bbp()
2048 decoder->state.items.is_32_bit = decoder->packet.count; in intel_pt_bbp()
2049 if (decoder->blk_type_pos < 0) { in intel_pt_bbp()
2051 decoder->blk_type); in intel_pt_bbp()
2052 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { in intel_pt_bbp()
2054 decoder->blk_type); in intel_pt_bbp()
2060 uint32_t id = decoder->packet.count; in intel_pt_bip()
2062 int pos = decoder->blk_type_pos; in intel_pt_bip()
2066 id, decoder->blk_type); in intel_pt_bip()
2070 if (decoder->state.items.mask[pos] & bit) { in intel_pt_bip()
2072 id, decoder->blk_type); in intel_pt_bip()
2075 decoder->state.items.mask[pos] |= bit; in intel_pt_bip()
2076 decoder->state.items.val[pos][id] = decoder->packet.payload; in intel_pt_bip()
2084 decoder->in_psb = true; in intel_pt_walk_psbend()
2091 switch (decoder->packet.type) { in intel_pt_walk_psbend()
2117 decoder->have_tma = false; in intel_pt_walk_psbend()
2119 err = -EAGAIN; in intel_pt_walk_psbend()
2147 decoder->pge = true; in intel_pt_walk_psbend()
2148 if (decoder->packet.count) { in intel_pt_walk_psbend()
2150 decoder->psb_ip = decoder->last_ip; in intel_pt_walk_psbend()
2160 if (decoder->period_type == INTEL_PT_PERIOD_MTC) in intel_pt_walk_psbend()
2161 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_walk_psbend()
2176 decoder->in_psb = false; in intel_pt_walk_psbend()
2185 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { in intel_pt_walk_fup_tip()
2186 decoder->tx_flags = 0; in intel_pt_walk_fup_tip()
2187 decoder->state.flags &= ~INTEL_PT_IN_TX; in intel_pt_walk_fup_tip()
2188 decoder->state.flags |= INTEL_PT_ABORT_TX; in intel_pt_walk_fup_tip()
2190 decoder->state.flags |= INTEL_PT_ASYNC; in intel_pt_walk_fup_tip()
2198 switch (decoder->packet.type) { in intel_pt_walk_fup_tip()
2223 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_fup_tip()
2224 decoder->pkt_step = 0; in intel_pt_walk_fup_tip()
2225 return -ENOENT; in intel_pt_walk_fup_tip()
2235 decoder->state.from_ip = decoder->ip; in intel_pt_walk_fup_tip()
2236 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
2237 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
2240 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
2242 decoder->pge = false; in intel_pt_walk_fup_tip()
2243 decoder->continuous_period = false; in intel_pt_walk_fup_tip()
2244 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_fup_tip()
2249 decoder->pge = true; in intel_pt_walk_fup_tip()
2251 decoder->ip); in intel_pt_walk_fup_tip()
2252 decoder->state.from_ip = 0; in intel_pt_walk_fup_tip()
2253 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
2254 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
2257 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
2259 decoder->state.type |= INTEL_PT_TRACE_BEGIN; in intel_pt_walk_fup_tip()
2265 decoder->state.from_ip = decoder->ip; in intel_pt_walk_fup_tip()
2266 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
2267 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
2270 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
2282 if (decoder->period_type == INTEL_PT_PERIOD_MTC) in intel_pt_walk_fup_tip()
2283 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_walk_fup_tip()
2307 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_resample()
2308 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_resample()
2309 decoder->state.from_ip = decoder->ip; in intel_pt_resample()
2310 decoder->state.to_ip = 0; in intel_pt_resample()
2327 struct intel_pt_vm_tsc_info *data = pkt_info->data; in intel_pt_vm_psb_lookahead_cb()
2329 switch (pkt_info->packet.type) { in intel_pt_vm_psb_lookahead_cb()
2341 data->tsc = true; in intel_pt_vm_psb_lookahead_cb()
2345 data->tma_packet = pkt_info->packet; in intel_pt_vm_psb_lookahead_cb()
2346 data->tma = true; in intel_pt_vm_psb_lookahead_cb()
2350 data->pip_packet = pkt_info->packet; in intel_pt_vm_psb_lookahead_cb()
2351 data->pip = true; in intel_pt_vm_psb_lookahead_cb()
2355 data->vmcs_packet = pkt_info->packet; in intel_pt_vm_psb_lookahead_cb()
2356 data->vmcs = true; in intel_pt_vm_psb_lookahead_cb()
2360 data->psbend = true; in intel_pt_vm_psb_lookahead_cb()
2400 struct intel_pt_ovf_fup_info *data = pkt_info->data; in intel_pt_ovf_fup_lookahead_cb()
2402 if (pkt_info->packet.type == INTEL_PT_CYC || in intel_pt_ovf_fup_lookahead_cb()
2403 pkt_info->packet.type == INTEL_PT_MTC || in intel_pt_ovf_fup_lookahead_cb()
2404 pkt_info->packet.type == INTEL_PT_TSC) in intel_pt_ovf_fup_lookahead_cb()
2405 return !--(data->max_lookahead); in intel_pt_ovf_fup_lookahead_cb()
2406 data->found = pkt_info->packet.type == INTEL_PT_FUP; in intel_pt_ovf_fup_lookahead_cb()
2424 struct intel_pt_vm_tsc_info *data = pkt_info->data; in intel_pt_tma_lookahead_cb()
2426 if (pkt_info->packet.type == INTEL_PT_CYC || in intel_pt_tma_lookahead_cb()
2427 pkt_info->packet.type == INTEL_PT_MTC) in intel_pt_tma_lookahead_cb()
2428 return !--(data->max_lookahead); in intel_pt_tma_lookahead_cb()
2430 if (pkt_info->packet.type == INTEL_PT_TMA) { in intel_pt_tma_lookahead_cb()
2431 data->tma_packet = pkt_info->packet; in intel_pt_tma_lookahead_cb()
2432 data->tma = true; in intel_pt_tma_lookahead_cb()
2439 if (decoder->tsc_ctc_mult) in intel_pt_ctc_to_tsc()
2440 return ctc * decoder->tsc_ctc_mult; in intel_pt_ctc_to_tsc()
2442 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); in intel_pt_ctc_to_tsc()
2457 * past an MTC, so a sign-extended value is used. in intel_pt_calc_expected_tsc()
2459 uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc); in intel_pt_calc_expected_tsc()
2489 uint32_t ctc = data->tma_packet.payload; in intel_pt_expected_tsc()
2490 uint32_t fc = data->tma_packet.count; in intel_pt_expected_tsc()
2493 decoder->ctc_timestamp, in intel_pt_expected_tsc()
2494 data->ctc_delta, data->last_ctc); in intel_pt_expected_tsc()
2500 uint64_t payload = decoder->packet.payload; in intel_pt_translate_vm_tsc()
2503 decoder->packet.payload -= vmcs_info->tsc_offset; in intel_pt_translate_vm_tsc()
2505 decoder->packet.payload &= SEVEN_BYTES; in intel_pt_translate_vm_tsc()
2511 if (!decoder->vm_tm_corr_dry_run) in intel_pt_translate_vm_tsc()
2512 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7); in intel_pt_translate_vm_tsc()
2514 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 in intel_pt_translate_vm_tsc()
2516 payload, decoder->packet.payload, vmcs_info->vmcs, in intel_pt_translate_vm_tsc()
2517 vmcs_info->tsc_offset); in intel_pt_translate_vm_tsc()
2538 return pip_packet->payload & 1; in pip_in_vm()
2544 vmcs_info->vmcs, vmcs_info->tsc_offset); in intel_pt_print_vmcs_info()
2551 data->ctc_delta = decoder->ctc_delta; in intel_pt_vm_tm_corr_psb()
2552 data->last_ctc = decoder->last_ctc; in intel_pt_vm_tm_corr_psb()
2554 if (data->tsc && !data->psbend) in intel_pt_vm_tm_corr_psb()
2556 decoder->in_psb = data->psbend; in intel_pt_vm_tm_corr_psb()
2564 if (!decoder->in_psb) { in intel_pt_vm_tm_corr_first_tsc()
2569 if (data->pip) { in intel_pt_vm_tm_corr_first_tsc()
2570 if (pip_in_vm(&data->pip_packet)) { /* Guest */ in intel_pt_vm_tm_corr_first_tsc()
2571 if (vmcs_info && vmcs_info->tsc_offset) { in intel_pt_vm_tm_corr_first_tsc()
2573 decoder->vm_tm_corr_reliable = true; in intel_pt_vm_tm_corr_first_tsc()
2578 decoder->vm_tm_corr_reliable = true; in intel_pt_vm_tm_corr_first_tsc()
2581 decoder->vm_tm_corr_reliable = false; in intel_pt_vm_tm_corr_first_tsc()
2586 if (vmcs_info && vmcs_info->tsc_offset) in intel_pt_vm_tm_corr_first_tsc()
2609 if (!decoder->in_psb) { in intel_pt_vm_tm_corr_tsc()
2611 data->ctc_delta = decoder->ctc_delta; in intel_pt_vm_tm_corr_tsc()
2612 data->last_ctc = decoder->last_ctc; in intel_pt_vm_tm_corr_tsc()
2613 data->max_lookahead = 16; in intel_pt_vm_tm_corr_tsc()
2615 if (decoder->pge) { in intel_pt_vm_tm_corr_tsc()
2616 data->pip = true; in intel_pt_vm_tm_corr_tsc()
2617 data->pip_packet.payload = decoder->pip_payload; in intel_pt_vm_tm_corr_tsc()
2622 if (!data->tma) { in intel_pt_vm_tm_corr_tsc()
2627 vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs; in intel_pt_vm_tm_corr_tsc()
2631 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); in intel_pt_vm_tm_corr_tsc()
2633 ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp; in intel_pt_vm_tm_corr_tsc()
2634 host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp); in intel_pt_vm_tm_corr_tsc()
2636 if (!decoder->ctc_timestamp) { in intel_pt_vm_tm_corr_tsc()
2643 tsc_offset = host_tsc - expected_tsc; in intel_pt_vm_tm_corr_tsc()
2646 if (data->pip) { in intel_pt_vm_tm_corr_tsc()
2647 if (pip_in_vm(&data->pip_packet)) { /* Guest */ in intel_pt_vm_tm_corr_tsc()
2652 decoder->vm_tm_corr_reliable = false; in intel_pt_vm_tm_corr_tsc()
2656 decoder->last_reliable_timestamp = host_tsc; in intel_pt_vm_tm_corr_tsc()
2657 decoder->vm_tm_corr_reliable = true; in intel_pt_vm_tm_corr_tsc()
2662 if (decoder->in_psb) { in intel_pt_vm_tm_corr_tsc()
2674 (vmcs_info->tsc_offset & SEVEN_BYTES)) { in intel_pt_vm_tm_corr_tsc()
2687 decoder->vm_tm_corr_reliable = false; in intel_pt_vm_tm_corr_tsc()
2695 if (vmcs_info && vmcs_info->vmcs) { in intel_pt_vm_tm_corr_tsc()
2696 if (vmcs_info->tsc_offset && vmcs_info->reliable) { in intel_pt_vm_tm_corr_tsc()
2698 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable && in intel_pt_vm_tm_corr_tsc()
2699 decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) { in intel_pt_vm_tm_corr_tsc()
2700 /* Continuous tracing, TSC in a PSB is not a time loss */ in intel_pt_vm_tm_corr_tsc()
2703 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) { in intel_pt_vm_tm_corr_tsc()
2705 * Unlikely to be a time loss TSC in a PSB which is not in intel_pt_vm_tm_corr_tsc()
2714 if (assign && (vmcs_info->tsc_offset != tsc_offset || in intel_pt_vm_tm_corr_tsc()
2715 vmcs_info->reliable != assign_reliable)) { in intel_pt_vm_tm_corr_tsc()
2716 bool print = vmcs_info->tsc_offset != tsc_offset; in intel_pt_vm_tm_corr_tsc()
2718 vmcs_info->tsc_offset = tsc_offset; in intel_pt_vm_tm_corr_tsc()
2719 vmcs_info->reliable = assign_reliable; in intel_pt_vm_tm_corr_tsc()
2725 if (vmcs_info && vmcs_info->tsc_offset) { in intel_pt_vm_tm_corr_tsc()
2726 if (!vmcs_info->reliable) in intel_pt_vm_tm_corr_tsc()
2732 if (!vmcs_info->error_printed) { in intel_pt_vm_tm_corr_tsc()
2734 vmcs_info->vmcs); in intel_pt_vm_tm_corr_tsc()
2735 vmcs_info->error_printed = true; in intel_pt_vm_tm_corr_tsc()
2744 decoder->vm_tm_corr_reliable = reliable; in intel_pt_vm_tm_corr_tsc()
2749 uint64_t host_tsc = decoder->packet.payload; in intel_pt_vm_tm_corr_pebs_tsc()
2750 uint64_t guest_tsc = decoder->packet.payload; in intel_pt_vm_tm_corr_pebs_tsc()
2754 vmcs = decoder->vmcs; in intel_pt_vm_tm_corr_pebs_tsc()
2758 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); in intel_pt_vm_tm_corr_pebs_tsc()
2760 if (decoder->pge) { in intel_pt_vm_tm_corr_pebs_tsc()
2761 if (in_vm(decoder->pip_payload)) { /* Guest */ in intel_pt_vm_tm_corr_pebs_tsc()
2778 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES; in intel_pt_vm_tm_corr_pebs_tsc()
2779 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp); in intel_pt_vm_tm_corr_pebs_tsc()
2780 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 in intel_pt_vm_tm_corr_pebs_tsc()
2782 guest_tsc, host_tsc, vmcs_info->vmcs, in intel_pt_vm_tm_corr_pebs_tsc()
2783 vmcs_info->tsc_offset); in intel_pt_vm_tm_corr_pebs_tsc()
2790 host_tsc = decoder->timestamp; in intel_pt_vm_tm_corr_pebs_tsc()
2793 decoder->packet.payload = host_tsc; in intel_pt_vm_tm_corr_pebs_tsc()
2795 if (!decoder->vm_tm_corr_dry_run) in intel_pt_vm_tm_corr_pebs_tsc()
2796 memcpy((void *)decoder->buf + 1, &host_tsc, 8); in intel_pt_vm_tm_corr_pebs_tsc()
2805 if (decoder->in_psb) in intel_pt_vm_time_correlation()
2810 if (err == -ENOLINK) in intel_pt_vm_time_correlation()
2815 switch (decoder->packet.type) { in intel_pt_vm_time_correlation()
2817 decoder->pge = false; in intel_pt_vm_time_correlation()
2818 decoder->vm_tm_corr_continuous = false; in intel_pt_vm_time_correlation()
2824 decoder->pge = true; in intel_pt_vm_time_correlation()
2828 decoder->in_psb = false; in intel_pt_vm_time_correlation()
2829 pge = decoder->pge; in intel_pt_vm_time_correlation()
2830 decoder->pge = intel_pt_ovf_fup_lookahead(decoder); in intel_pt_vm_time_correlation()
2831 if (pge != decoder->pge) in intel_pt_vm_time_correlation()
2833 if (!decoder->pge) in intel_pt_vm_time_correlation()
2834 decoder->vm_tm_corr_continuous = false; in intel_pt_vm_time_correlation()
2838 if (decoder->in_psb) in intel_pt_vm_time_correlation()
2839 decoder->pge = true; in intel_pt_vm_time_correlation()
2843 decoder->pge = false; in intel_pt_vm_time_correlation()
2844 decoder->vm_tm_corr_continuous = false; in intel_pt_vm_time_correlation()
2845 decoder->have_tma = false; in intel_pt_vm_time_correlation()
2853 decoder->pip_payload = decoder->packet.payload; in intel_pt_vm_time_correlation()
2863 decoder->vm_tm_corr_same_buf = true; in intel_pt_vm_time_correlation()
2864 decoder->vm_tm_corr_continuous = decoder->pge; in intel_pt_vm_time_correlation()
2880 decoder->in_psb = false; in intel_pt_vm_time_correlation()
2885 if (decoder->packet.payload != NO_VMCS) in intel_pt_vm_time_correlation()
2886 decoder->vmcs = decoder->packet.payload; in intel_pt_vm_time_correlation()
2890 decoder->blk_type = decoder->packet.payload; in intel_pt_vm_time_correlation()
2894 if (decoder->blk_type == INTEL_PT_PEBS_BASIC && in intel_pt_vm_time_correlation()
2895 decoder->packet.count == 2) in intel_pt_vm_time_correlation()
2901 decoder->blk_type = 0; in intel_pt_vm_time_correlation()
2940 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { in intel_pt_hop_trace()
2946 switch (decoder->packet.type) { in intel_pt_hop_trace()
2951 decoder->pge = false; in intel_pt_hop_trace()
2952 if (!decoder->packet.count) { in intel_pt_hop_trace()
2957 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_hop_trace()
2958 decoder->state.from_ip = 0; in intel_pt_hop_trace()
2959 decoder->state.to_ip = decoder->ip; in intel_pt_hop_trace()
2964 if (!decoder->packet.count) { in intel_pt_hop_trace()
2969 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_hop_trace()
2970 decoder->state.from_ip = decoder->ip; in intel_pt_hop_trace()
2971 decoder->state.to_ip = 0; in intel_pt_hop_trace()
2977 if (!decoder->packet.count) in intel_pt_hop_trace()
2980 if (decoder->set_fup_mwait || decoder->set_fup_pwre) in intel_pt_hop_trace()
2982 if (!decoder->branch_enable || !decoder->pge) in intel_pt_hop_trace()
2985 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_hop_trace()
2986 decoder->state.from_ip = decoder->ip; in intel_pt_hop_trace()
2987 decoder->state.to_ip = 0; in intel_pt_hop_trace()
2992 decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH; in intel_pt_hop_trace()
2994 if (!*err && decoder->state.to_ip) in intel_pt_hop_trace()
2995 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_hop_trace()
2999 decoder->state.psb_offset = decoder->pos; in intel_pt_hop_trace()
3000 decoder->psb_ip = 0; in intel_pt_hop_trace()
3001 decoder->last_ip = 0; in intel_pt_hop_trace()
3002 decoder->have_last_ip = true; in intel_pt_hop_trace()
3004 if (*err == -EAGAIN) in intel_pt_hop_trace()
3008 decoder->state.type = INTEL_PT_PSB_EVT; in intel_pt_hop_trace()
3009 if (decoder->psb_ip) { in intel_pt_hop_trace()
3010 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_hop_trace()
3011 decoder->ip = decoder->psb_ip; in intel_pt_hop_trace()
3013 decoder->state.from_ip = decoder->psb_ip; in intel_pt_hop_trace()
3014 decoder->state.to_ip = 0; in intel_pt_hop_trace()
3061 struct intel_pt_psb_info *data = pkt_info->data; in intel_pt_psb_lookahead_cb()
3063 switch (pkt_info->packet.type) { in intel_pt_psb_lookahead_cb()
3075 if (data->after_psbend) { in intel_pt_psb_lookahead_cb()
3076 data->after_psbend -= 1; in intel_pt_psb_lookahead_cb()
3077 if (!data->after_psbend) in intel_pt_psb_lookahead_cb()
3083 if (data->after_psbend) in intel_pt_psb_lookahead_cb()
3085 if (data->fup || pkt_info->packet.count == 0) in intel_pt_psb_lookahead_cb()
3087 data->fup_packet = pkt_info->packet; in intel_pt_psb_lookahead_cb()
3088 data->fup = true; in intel_pt_psb_lookahead_cb()
3092 if (!data->fup) in intel_pt_psb_lookahead_cb()
3095 data->after_psbend = 6; in intel_pt_psb_lookahead_cb()
3100 if (data->after_psbend) in intel_pt_psb_lookahead_cb()
3101 data->fup = false; in intel_pt_psb_lookahead_cb()
3118 if (data->after_psbend) { in intel_pt_psb_lookahead_cb()
3119 data->after_psbend -= 1; in intel_pt_psb_lookahead_cb()
3120 if (!data->after_psbend) in intel_pt_psb_lookahead_cb()
3144 decoder->last_ip = 0; in intel_pt_psb()
3145 decoder->psb_ip = 0; in intel_pt_psb()
3146 decoder->have_last_ip = true; in intel_pt_psb()
3147 intel_pt_clear_stack(&decoder->stack); in intel_pt_psb()
3151 decoder->state.type = INTEL_PT_PSB_EVT; in intel_pt_psb()
3152 decoder->state.from_ip = decoder->psb_ip; in intel_pt_psb()
3153 decoder->state.to_ip = 0; in intel_pt_psb()
3161 if (decoder->ip != decoder->last_ip) { in intel_pt_fup_in_psb()
3163 if (!err || err != -EAGAIN) in intel_pt_fup_in_psb()
3167 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_fup_in_psb()
3170 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_fup_in_psb()
3171 return -ENOENT; in intel_pt_fup_in_psb()
3181 if (!decoder->branch_enable) in intel_pt_psb_with_fup()
3188 decoder->packet = data.fup_packet; in intel_pt_psb_with_fup()
3190 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB; in intel_pt_psb_with_fup()
3209 if (decoder->cyc_threshold) { in intel_pt_walk_trace()
3210 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) in intel_pt_walk_trace()
3211 decoder->sample_cyc = false; in intel_pt_walk_trace()
3212 last_packet_type = decoder->packet.type; in intel_pt_walk_trace()
3215 if (decoder->hop) { in intel_pt_walk_trace()
3228 switch (decoder->packet.type) { in intel_pt_walk_trace()
3230 if (!decoder->packet.count) in intel_pt_walk_trace()
3232 decoder->tnt = decoder->packet; in intel_pt_walk_trace()
3233 decoder->pkt_state = INTEL_PT_STATE_TNT; in intel_pt_walk_trace()
3235 if (err == -EAGAIN) in intel_pt_walk_trace()
3240 if (decoder->packet.count != 0) in intel_pt_walk_trace()
3242 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; in intel_pt_walk_trace()
3246 decoder->pge = true; in intel_pt_walk_trace()
3247 decoder->overflow = false; in intel_pt_walk_trace()
3250 if (decoder->packet.count == 0) { in intel_pt_walk_trace()
3252 decoder->pos); in intel_pt_walk_trace()
3257 decoder->state.from_ip = 0; in intel_pt_walk_trace()
3258 decoder->state.to_ip = decoder->ip; in intel_pt_walk_trace()
3259 decoder->state.type |= INTEL_PT_TRACE_BEGIN; in intel_pt_walk_trace()
3264 if (decoder->hop) in intel_pt_walk_trace()
3265 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_walk_trace()
3273 if (decoder->packet.count != 0) in intel_pt_walk_trace()
3275 decoder->pkt_state = INTEL_PT_STATE_TIP; in intel_pt_walk_trace()
3279 if (decoder->packet.count == 0) { in intel_pt_walk_trace()
3281 decoder->pos); in intel_pt_walk_trace()
3286 if (!decoder->branch_enable || !decoder->pge) { in intel_pt_walk_trace()
3287 decoder->ip = decoder->last_ip; in intel_pt_walk_trace()
3293 if (decoder->set_fup_mwait) in intel_pt_walk_trace()
3296 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; in intel_pt_walk_trace()
3298 decoder->pkt_state = INTEL_PT_STATE_FUP; in intel_pt_walk_trace()
3300 if (err != -EAGAIN) in intel_pt_walk_trace()
3309 decoder->pge = false; in intel_pt_walk_trace()
3310 decoder->continuous_period = false; in intel_pt_walk_trace()
3312 decoder->have_tma = false; in intel_pt_walk_trace()
3316 decoder->state.psb_offset = decoder->pos; in intel_pt_walk_trace()
3317 decoder->psb_ip = 0; in intel_pt_walk_trace()
3321 if (err == -EAGAIN) in intel_pt_walk_trace()
3331 if (decoder->period_type != INTEL_PT_PERIOD_MTC) in intel_pt_walk_trace()
3337 if (!decoder->mtc_insn) in intel_pt_walk_trace()
3339 decoder->mtc_insn = false; in intel_pt_walk_trace()
3341 if (!decoder->timestamp) in intel_pt_walk_trace()
3343 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_trace()
3344 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
3345 decoder->state.to_ip = 0; in intel_pt_walk_trace()
3346 decoder->mtc_insn = false; in intel_pt_walk_trace()
3363 if (decoder->cbr != decoder->cbr_seen) { in intel_pt_walk_trace()
3364 decoder->state.type = 0; in intel_pt_walk_trace()
3374 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
3375 decoder->set_fup_mode_exec = true; in intel_pt_walk_trace()
3382 if (!decoder->pge || decoder->in_psb) { in intel_pt_walk_trace()
3401 decoder->fup_ptw_payload = decoder->packet.payload; in intel_pt_walk_trace()
3405 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
3406 decoder->set_fup_ptw = true; in intel_pt_walk_trace()
3410 decoder->pos); in intel_pt_walk_trace()
3415 decoder->state.type = INTEL_PT_PTW; in intel_pt_walk_trace()
3416 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
3417 decoder->state.to_ip = 0; in intel_pt_walk_trace()
3418 decoder->state.ptw_payload = decoder->packet.payload; in intel_pt_walk_trace()
3422 decoder->fup_mwait_payload = decoder->packet.payload; in intel_pt_walk_trace()
3423 decoder->set_fup_mwait = true; in intel_pt_walk_trace()
3427 if (decoder->set_fup_mwait) { in intel_pt_walk_trace()
3428 decoder->fup_pwre_payload = in intel_pt_walk_trace()
3429 decoder->packet.payload; in intel_pt_walk_trace()
3430 decoder->set_fup_pwre = true; in intel_pt_walk_trace()
3433 decoder->state.type = INTEL_PT_PWR_ENTRY; in intel_pt_walk_trace()
3434 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
3435 decoder->state.to_ip = 0; in intel_pt_walk_trace()
3436 decoder->state.pwrx_payload = decoder->packet.payload; in intel_pt_walk_trace()
3443 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
3444 decoder->set_fup_exstop = true; in intel_pt_walk_trace()
3448 decoder->pos); in intel_pt_walk_trace()
3453 decoder->state.type = INTEL_PT_EX_STOP; in intel_pt_walk_trace()
3454 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
3455 decoder->state.to_ip = 0; in intel_pt_walk_trace()
3459 decoder->state.type = INTEL_PT_PWR_EXIT; in intel_pt_walk_trace()
3460 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
3461 decoder->state.to_ip = 0; in intel_pt_walk_trace()
3462 decoder->state.pwrx_payload = decoder->packet.payload; in intel_pt_walk_trace()
3474 decoder->state.type = INTEL_PT_BLK_ITEMS; in intel_pt_walk_trace()
3475 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
3476 decoder->state.to_ip = 0; in intel_pt_walk_trace()
3483 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
3484 decoder->set_fup_bep = true; in intel_pt_walk_trace()
3488 decoder->pos); in intel_pt_walk_trace()
3493 decoder->fup_cfe_pkt = decoder->packet; in intel_pt_walk_trace()
3494 decoder->set_fup_cfe = true; in intel_pt_walk_trace()
3495 if (!decoder->pge) { in intel_pt_walk_trace()
3502 decoder->fup_cfe_pkt = decoder->packet; in intel_pt_walk_trace()
3506 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
3507 decoder->set_fup_cfe_ip = true; in intel_pt_walk_trace()
3511 decoder->pos); in intel_pt_walk_trace()
3529 return decoder->packet.count && in intel_pt_have_ip()
3530 (decoder->have_last_ip || decoder->packet.count == 3 || in intel_pt_have_ip()
3531 decoder->packet.count == 6); in intel_pt_have_ip()
3539 decoder->in_psb = true; in intel_pt_walk_psb()
3546 switch (decoder->packet.type) { in intel_pt_walk_psb()
3548 decoder->continuous_period = false; in intel_pt_walk_psb()
3567 err = -ENOENT; in intel_pt_walk_psb()
3571 decoder->pge = true; in intel_pt_walk_psb()
3573 uint64_t current_ip = decoder->ip; in intel_pt_walk_psb()
3576 decoder->psb_ip = decoder->ip; in intel_pt_walk_psb()
3579 decoder->ip); in intel_pt_walk_psb()
3616 decoder->pge = false; in intel_pt_walk_psb()
3617 decoder->continuous_period = false; in intel_pt_walk_psb()
3622 decoder->have_tma = false; in intel_pt_walk_psb()
3624 if (decoder->ip) in intel_pt_walk_psb()
3625 decoder->pkt_state = INTEL_PT_STATE_ERR4; in intel_pt_walk_psb()
3627 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_psb()
3628 err = -ENOENT; in intel_pt_walk_psb()
3652 decoder->in_psb = false; in intel_pt_walk_psb()
3666 switch (decoder->packet.type) { in intel_pt_walk_to_ip()
3668 decoder->continuous_period = false; in intel_pt_walk_to_ip()
3669 decoder->pge = false; in intel_pt_walk_to_ip()
3672 if (!decoder->ip) in intel_pt_walk_to_ip()
3674 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_to_ip()
3678 decoder->pge = true; in intel_pt_walk_to_ip()
3682 if (!decoder->ip) in intel_pt_walk_to_ip()
3684 decoder->state.type |= INTEL_PT_TRACE_BEGIN; in intel_pt_walk_to_ip()
3688 decoder->pge = true; in intel_pt_walk_to_ip()
3691 if (!decoder->ip) in intel_pt_walk_to_ip()
3698 if (decoder->ip) in intel_pt_walk_to_ip()
3741 decoder->pge = false; in intel_pt_walk_to_ip()
3742 decoder->continuous_period = false; in intel_pt_walk_to_ip()
3744 decoder->have_tma = false; in intel_pt_walk_to_ip()
3748 decoder->state.psb_offset = decoder->pos; in intel_pt_walk_to_ip()
3749 decoder->psb_ip = 0; in intel_pt_walk_to_ip()
3750 decoder->last_ip = 0; in intel_pt_walk_to_ip()
3751 decoder->have_last_ip = true; in intel_pt_walk_to_ip()
3752 intel_pt_clear_stack(&decoder->stack); in intel_pt_walk_to_ip()
3756 decoder->state.type = INTEL_PT_PSB_EVT; in intel_pt_walk_to_ip()
3757 decoder->state.from_ip = decoder->psb_ip; in intel_pt_walk_to_ip()
3758 decoder->state.to_ip = 0; in intel_pt_walk_to_ip()
3791 decoder->overflow = false; in intel_pt_sync_ip()
3793 if (!decoder->branch_enable) { in intel_pt_sync_ip()
3794 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync_ip()
3795 decoder->state.type = 0; /* Do not have a sample */ in intel_pt_sync_ip()
3801 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) in intel_pt_sync_ip()
3805 if (decoder->hop) in intel_pt_sync_ip()
3806 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_sync_ip()
3808 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync_ip()
3810 decoder->state.from_ip = 0; in intel_pt_sync_ip()
3811 decoder->state.to_ip = decoder->ip; in intel_pt_sync_ip()
3812 intel_pt_log_to("Setting IP", decoder->ip); in intel_pt_sync_ip()
3819 const unsigned char *end = decoder->buf + decoder->len; in intel_pt_part_psb()
3822 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { in intel_pt_part_psb()
3823 if (i > decoder->len) in intel_pt_part_psb()
3825 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) in intel_pt_part_psb()
3833 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; in intel_pt_rest_psb()
3836 if (rest_psb > decoder->len || in intel_pt_rest_psb()
3837 memcmp(decoder->buf, psb + part_psb, rest_psb)) in intel_pt_rest_psb()
3848 decoder->pos += decoder->len; in intel_pt_get_split_psb()
3849 decoder->len = 0; in intel_pt_get_split_psb()
3859 decoder->pos -= part_psb; in intel_pt_get_split_psb()
3860 decoder->next_buf = decoder->buf + rest_psb; in intel_pt_get_split_psb()
3861 decoder->next_len = decoder->len - rest_psb; in intel_pt_get_split_psb()
3862 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); in intel_pt_get_split_psb()
3863 decoder->buf = decoder->temp_buf; in intel_pt_get_split_psb()
3864 decoder->len = INTEL_PT_PSB_LEN; in intel_pt_get_split_psb()
3876 if (!decoder->len) { in intel_pt_scan_for_psb()
3882 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, in intel_pt_scan_for_psb()
3893 decoder->pos += decoder->len; in intel_pt_scan_for_psb()
3894 decoder->len = 0; in intel_pt_scan_for_psb()
3899 decoder->pkt_step = next - decoder->buf; in intel_pt_scan_for_psb()
3908 decoder->pge = false; in intel_pt_sync()
3909 decoder->continuous_period = false; in intel_pt_sync()
3910 decoder->have_last_ip = false; in intel_pt_sync()
3911 decoder->last_ip = 0; in intel_pt_sync()
3912 decoder->psb_ip = 0; in intel_pt_sync()
3913 decoder->ip = 0; in intel_pt_sync()
3914 intel_pt_clear_stack(&decoder->stack); in intel_pt_sync()
3920 if (decoder->vm_time_correlation) { in intel_pt_sync()
3921 decoder->in_psb = true; in intel_pt_sync()
3922 if (!decoder->timestamp) in intel_pt_sync()
3923 decoder->timestamp = 1; in intel_pt_sync()
3924 decoder->state.type = 0; in intel_pt_sync()
3925 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION; in intel_pt_sync()
3929 decoder->have_last_ip = true; in intel_pt_sync()
3930 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync()
3936 decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ in intel_pt_sync()
3937 decoder->state.from_ip = decoder->psb_ip; in intel_pt_sync()
3938 decoder->state.to_ip = 0; in intel_pt_sync()
3940 if (decoder->ip) { in intel_pt_sync()
3945 if (decoder->hop) in intel_pt_sync()
3946 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_sync()
3948 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync()
3956 uint64_t est = decoder->sample_insn_cnt << 1; in intel_pt_est_timestamp()
3958 if (!decoder->cbr || !decoder->max_non_turbo_ratio) in intel_pt_est_timestamp()
3961 est *= decoder->max_non_turbo_ratio; in intel_pt_est_timestamp()
3962 est /= decoder->cbr; in intel_pt_est_timestamp()
3964 return decoder->sample_timestamp + est; in intel_pt_est_timestamp()
3972 decoder->state.type = INTEL_PT_BRANCH; in intel_pt_decode()
3973 decoder->state.flags = 0; in intel_pt_decode()
3975 switch (decoder->pkt_state) { in intel_pt_decode()
3980 decoder->have_last_ip = false; in intel_pt_decode()
3981 decoder->last_ip = 0; in intel_pt_decode()
3982 decoder->ip = 0; in intel_pt_decode()
3993 if (err == -EAGAIN) in intel_pt_decode()
4002 if (err == -EAGAIN) in intel_pt_decode()
4007 if (err == -EAGAIN) in intel_pt_decode()
4023 } while (err == -ENOLINK); in intel_pt_decode()
4026 decoder->state.err = intel_pt_ext_err(err); in intel_pt_decode()
4027 if (err != -EOVERFLOW) in intel_pt_decode()
4028 decoder->state.from_ip = decoder->ip; in intel_pt_decode()
4030 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; in intel_pt_decode()
4033 decoder->state.err = 0; in intel_pt_decode()
4034 if (decoder->cbr != decoder->cbr_seen) { in intel_pt_decode()
4035 decoder->cbr_seen = decoder->cbr; in intel_pt_decode()
4036 if (!decoder->state.type) { in intel_pt_decode()
4037 decoder->state.from_ip = decoder->ip; in intel_pt_decode()
4038 decoder->state.to_ip = 0; in intel_pt_decode()
4040 decoder->state.type |= INTEL_PT_CBR_CHG; in intel_pt_decode()
4041 decoder->state.cbr_payload = decoder->cbr_payload; in intel_pt_decode()
4042 decoder->state.cbr = decoder->cbr; in intel_pt_decode()
4044 if (intel_pt_sample_time(decoder->pkt_state)) { in intel_pt_decode()
4046 if (decoder->sample_cyc) { in intel_pt_decode()
4047 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; in intel_pt_decode()
4048 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; in intel_pt_decode()
4049 decoder->sample_cyc = false; in intel_pt_decode()
4056 if (!decoder->have_cyc) in intel_pt_decode()
4057 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; in intel_pt_decode()
4061 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) in intel_pt_decode()
4062 decoder->sample_timestamp = decoder->tsc_timestamp; in intel_pt_decode()
4064 decoder->state.from_nr = decoder->nr; in intel_pt_decode()
4065 decoder->state.to_nr = decoder->next_nr; in intel_pt_decode()
4066 decoder->nr = decoder->next_nr; in intel_pt_decode()
4068 decoder->state.timestamp = decoder->sample_timestamp; in intel_pt_decode()
4069 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); in intel_pt_decode()
4070 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; in intel_pt_decode()
4071 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; in intel_pt_decode()
4073 return &decoder->state; in intel_pt_decode()
4077 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
4093 *len -= next - *buf; in intel_pt_next_psb()
4101 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
4119 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); in intel_pt_step_psb()
4121 *len -= next - *buf; in intel_pt_step_psb()
4129 * intel_pt_last_psb - find the last PSB packet in a buffer.
4146 k = len - INTEL_PT_PSB_LEN + 1; in intel_pt_last_psb()
4151 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) in intel_pt_last_psb()
4153 k = p - buf; in intel_pt_last_psb()
4160 * intel_pt_next_tsc - find and return next TSC.
4191 len -= ret; in intel_pt_next_tsc()
4197 * intel_pt_tsc_cmp - compare 7-byte TSCs.
4201 * This function compares 7-byte TSC values allowing for the possibility that
4206 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
4217 if (tsc2 - tsc1 < halfway) in intel_pt_tsc_cmp()
4218 return -1; in intel_pt_tsc_cmp()
4222 if (tsc1 - tsc2 < halfway) in intel_pt_tsc_cmp()
4225 return -1; in intel_pt_tsc_cmp()
4229 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
4232 * adj_for_padding - adjust overlap to account for padding.
4237 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
4240 * Return: A pointer into @buf_b from where non-overlapped data starts
4245 unsigned char *p = buf_b - MAX_PADDING; in adj_for_padding()
4246 unsigned char *q = buf_a + len_a - MAX_PADDING; in adj_for_padding()
4249 for (i = MAX_PADDING; i; i--, p++, q++) { in adj_for_padding()
4258 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
4266 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4269 * first TSC of @buf_b in order to determine if the buffers overlap, and then
4273 * Return: A pointer into @buf_b from where non-overlapped data starts, or
4274 * @buf_b + @len_b if there is no non-overlapped data.
4288 return buf_b; /* No PSB in buf_a => no overlap */ in intel_pt_find_overlap_tsc()
4290 len = len_a - (p - buf_a); in intel_pt_find_overlap_tsc()
4293 len_a -= len; in intel_pt_find_overlap_tsc()
4296 return buf_b; /* No full PSB+ => assume no overlap */ in intel_pt_find_overlap_tsc()
4297 len = len_a - (p - buf_a); in intel_pt_find_overlap_tsc()
4299 return buf_b; /* No TSC in buf_a => assume no overlap */ in intel_pt_find_overlap_tsc()
4312 start = buf_b + len_b - (rem_b - rem_a); in intel_pt_find_overlap_tsc()
4316 return buf_b; /* tsc_a < tsc_b => no overlap */ in intel_pt_find_overlap_tsc()
4325 * intel_pt_find_overlap - determine start of non-overlapped trace data.
4330 * @have_tsc: can use TSC packets to detect overlap
4333 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4339 * Return: A pointer into @buf_b from where non-overlapped data starts, or
4340 * @buf_b + @len_b if there is no non-overlapped data.
4354 return buf_b; /* No overlap */ in intel_pt_find_overlap()
4369 return buf_b; /* No overlap */ in intel_pt_find_overlap()
4374 /* Potential overlap so check the bytes */ in intel_pt_find_overlap()
4383 return buf_b; /* No overlap */ in intel_pt_find_overlap()
4388 * struct fast_forward_data - data used by intel_pt_ff_cb().
4399 * intel_pt_ff_cb - fast forward lookahead callback.
4416 buf = (unsigned char *)buffer->buf; in intel_pt_ff_cb()
4417 len = buffer->len; in intel_pt_ff_cb()
4423 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); in intel_pt_ff_cb()
4426 tsc, buffer->ref_timestamp); in intel_pt_ff_cb()
4432 if (tsc < d->timestamp) in intel_pt_ff_cb()
4433 d->buf_timestamp = buffer->ref_timestamp; in intel_pt_ff_cb()
4441 * intel_pt_fast_forward - reposition decoder forwards.
4459 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); in intel_pt_fast_forward()
4466 decoder->pos += decoder->len; in intel_pt_fast_forward()
4467 decoder->len = 0; in intel_pt_fast_forward()
4469 /* -ENOLINK means non-consecutive trace */ in intel_pt_fast_forward()
4470 if (err && err != -ENOLINK) in intel_pt_fast_forward()
4472 } while (decoder->buf_timestamp != d.buf_timestamp); in intel_pt_fast_forward()
4475 if (!decoder->buf) in intel_pt_fast_forward()
4478 buf = (unsigned char *)decoder->buf; in intel_pt_fast_forward()
4479 len = decoder->len; in intel_pt_fast_forward()
4494 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); in intel_pt_fast_forward()
4503 decoder->pos += decoder->len - len; in intel_pt_fast_forward()
4504 decoder->buf = buf; in intel_pt_fast_forward()
4505 decoder->len = len; in intel_pt_fast_forward()