1 /****************************************************************************
2 * Driver for Solarflare network controllers and boards
3 * Copyright 2011-2013 Solarflare Communications Inc.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
10 /* Theory of operation:
12 * PTP support is assisted by firmware running on the MC, which provides
13 * the hardware timestamping capabilities. Both transmitted and received
14 * PTP event packets are queued onto internal queues for subsequent processing;
15 * this is because the MC operations are relatively long and would block
16 * block NAPI/interrupt operation.
18 * Receive event processing:
19 * The event contains the packet's UUID and sequence number, together
20 * with the hardware timestamp. The PTP receive packet queue is searched
21 * for this UUID/sequence number and, if found, put on a pending queue.
22 * Packets not matching are delivered without timestamps (MCDI events will
23 * always arrive after the actual packet).
24 * It is important for the operation of the PTP protocol that the ordering
25 * of packets between the event and general port is maintained.
27 * Work queue processing:
28 * If work waiting, synchronise host/hardware time
30 * Transmit: send packet through MC, which returns the transmission time
31 * that is converted to an appropriate timestamp.
33 * Receive: the packet's reception time is converted to an appropriate
37 #include <linux/udp.h>
38 #include <linux/time.h>
39 #include <linux/ktime.h>
40 #include <linux/module.h>
41 #include <linux/net_tstamp.h>
42 #include <linux/pps_kernel.h>
43 #include <linux/ptp_clock_kernel.h>
44 #include "net_driver.h"
47 #include "mcdi_pcol.h"
49 #include "farch_regs.h"
52 /* Maximum number of events expected to make up a PTP event */
53 #define MAX_EVENT_FRAGS 3
55 /* Maximum delay, ms, to begin synchronisation */
56 #define MAX_SYNCHRONISE_WAIT_MS 2
58 /* How long, at most, to spend synchronising */
59 #define SYNCHRONISE_PERIOD_NS 250000
61 /* How often to update the shared memory time */
62 #define SYNCHRONISATION_GRANULARITY_NS 200
64 /* Minimum permitted length of a (corrected) synchronisation time */
65 #define DEFAULT_MIN_SYNCHRONISATION_NS 120
67 /* Maximum permitted length of a (corrected) synchronisation time */
68 #define MAX_SYNCHRONISATION_NS 1000
70 /* How many (MC) receive events that can be queued */
71 #define MAX_RECEIVE_EVENTS 8
73 /* Length of (modified) moving average. */
74 #define AVERAGE_LENGTH 16
76 /* How long an unmatched event or packet can be held */
77 #define PKT_EVENT_LIFETIME_MS 10
79 /* Offsets into PTP packet for identification. These offsets are from the
80 * start of the IP header, not the MAC header. Note that neither PTP V1 nor
81 * PTP V2 permit the use of IPV4 options.
83 #define PTP_DPORT_OFFSET 22
85 #define PTP_V1_VERSION_LENGTH 2
86 #define PTP_V1_VERSION_OFFSET 28
88 #define PTP_V1_UUID_LENGTH 6
89 #define PTP_V1_UUID_OFFSET 50
91 #define PTP_V1_SEQUENCE_LENGTH 2
92 #define PTP_V1_SEQUENCE_OFFSET 58
94 /* The minimum length of a PTP V1 packet for offsets, etc. to be valid:
97 #define PTP_V1_MIN_LENGTH 64
99 #define PTP_V2_VERSION_LENGTH 1
100 #define PTP_V2_VERSION_OFFSET 29
102 #define PTP_V2_UUID_LENGTH 8
103 #define PTP_V2_UUID_OFFSET 48
105 /* Although PTP V2 UUIDs are comprised a ClockIdentity (8) and PortNumber (2),
106 * the MC only captures the last six bytes of the clock identity. These values
107 * reflect those, not the ones used in the standard. The standard permits
108 * mapping of V1 UUIDs to V2 UUIDs with these same values.
110 #define PTP_V2_MC_UUID_LENGTH 6
111 #define PTP_V2_MC_UUID_OFFSET 50
113 #define PTP_V2_SEQUENCE_LENGTH 2
114 #define PTP_V2_SEQUENCE_OFFSET 58
116 /* The minimum length of a PTP V2 packet for offsets, etc. to be valid:
117 * includes IP header.
119 #define PTP_V2_MIN_LENGTH 63
121 #define PTP_MIN_LENGTH 63
123 #define PTP_ADDRESS 0xe0000181 /* 224.0.1.129 */
124 #define PTP_EVENT_PORT 319
125 #define PTP_GENERAL_PORT 320
127 /* Annoyingly the format of the version numbers are different between
128 * versions 1 and 2 so it isn't possible to simply look for 1 or 2.
130 #define PTP_VERSION_V1 1
132 #define PTP_VERSION_V2 2
133 #define PTP_VERSION_V2_MASK 0x0f
135 enum ptp_packet_state
{
136 PTP_PACKET_STATE_UNMATCHED
= 0,
137 PTP_PACKET_STATE_MATCHED
,
138 PTP_PACKET_STATE_TIMED_OUT
,
139 PTP_PACKET_STATE_MATCH_UNWANTED
142 /* NIC synchronised with single word of time only comprising
143 * partial seconds and full nanoseconds: 10^9 ~ 2^30 so 2 bits for seconds.
145 #define MC_NANOSECOND_BITS 30
146 #define MC_NANOSECOND_MASK ((1 << MC_NANOSECOND_BITS) - 1)
147 #define MC_SECOND_MASK ((1 << (32 - MC_NANOSECOND_BITS)) - 1)
149 /* Maximum parts-per-billion adjustment that is acceptable */
150 #define MAX_PPB 1000000
152 /* Number of bits required to hold the above */
153 #define MAX_PPB_BITS 20
155 /* Number of extra bits allowed when calculating fractional ns.
156 * EXTRA_BITS + MC_CMD_PTP_IN_ADJUST_BITS + MAX_PPB_BITS should
159 #define PPB_EXTRA_BITS 2
161 /* Precalculate scale word to avoid long long division at runtime */
162 #define PPB_SCALE_WORD ((1LL << (PPB_EXTRA_BITS + MC_CMD_PTP_IN_ADJUST_BITS +\
163 MAX_PPB_BITS)) / 1000000000LL)
165 #define PTP_SYNC_ATTEMPTS 4
168 * struct efx_ptp_match - Matching structure, stored in sk_buff's cb area.
169 * @words: UUID and (partial) sequence number
170 * @expiry: Time after which the packet should be delivered irrespective of
172 * @state: The state of the packet - whether it is ready for processing or
173 * whether that is of no interest.
175 struct efx_ptp_match
{
176 u32 words
[DIV_ROUND_UP(PTP_V1_UUID_LENGTH
, 4)];
177 unsigned long expiry
;
178 enum ptp_packet_state state
;
182 * struct efx_ptp_event_rx - A PTP receive event (from MC)
183 * @seq0: First part of (PTP) UUID
184 * @seq1: Second part of (PTP) UUID and sequence number
185 * @hwtimestamp: Event timestamp
187 struct efx_ptp_event_rx
{
188 struct list_head link
;
192 unsigned long expiry
;
196 * struct efx_ptp_timeset - Synchronisation between host and MC
197 * @host_start: Host time immediately before hardware timestamp taken
198 * @major: Hardware timestamp, major
199 * @minor: Hardware timestamp, minor
200 * @host_end: Host time immediately after hardware timestamp taken
201 * @wait: Number of NIC clock ticks between hardware timestamp being read and
202 * host end time being seen
203 * @window: Difference of host_end and host_start
204 * @valid: Whether this timeset is valid
206 struct efx_ptp_timeset
{
212 u32 window
; /* Derived: end - start, allowing for wrap */
216 * struct efx_ptp_data - Precision Time Protocol (PTP) state
217 * @efx: The NIC context
218 * @channel: The PTP channel (Siena only)
219 * @rx_ts_inline: Flag for whether RX timestamps are inline (else they are
221 * @rxq: Receive queue (awaiting timestamps)
222 * @txq: Transmit queue
223 * @evt_list: List of MC receive events awaiting packets
224 * @evt_free_list: List of free events
225 * @evt_lock: Lock for manipulating evt_list and evt_free_list
226 * @evt_overflow: Boolean indicating that event list has overflowed
227 * @rx_evts: Instantiated events (on evt_list and evt_free_list)
228 * @workwq: Work queue for processing pending PTP operations
230 * @reset_required: A serious error has occurred and the PTP task needs to be
231 * reset (disable, enable).
232 * @rxfilter_event: Receive filter when operating
233 * @rxfilter_general: Receive filter when operating
234 * @config: Current timestamp configuration
235 * @enabled: PTP operation enabled
236 * @mode: Mode in which PTP operating (PTP version)
237 * @time_format: Time format supported by this NIC
238 * @ns_to_nic_time: Function to convert from scalar nanoseconds to NIC time
239 * @nic_to_kernel_time: Function to convert from NIC to kernel time
240 * @min_synchronisation_ns: Minimum acceptable corrected sync window
241 * @ts_corrections.tx: Required driver correction of transmit timestamps
242 * @ts_corrections.rx: Required driver correction of receive timestamps
243 * @ts_corrections.pps_out: PPS output error (information only)
244 * @ts_corrections.pps_in: Required driver correction of PPS input timestamps
245 * @evt_frags: Partly assembled PTP events
246 * @evt_frag_idx: Current fragment number
247 * @evt_code: Last event code
248 * @start: Address at which MC indicates ready for synchronisation
249 * @host_time_pps: Host time at last PPS
250 * @current_adjfreq: Current ppb adjustment.
251 * @phc_clock: Pointer to registered phc device
252 * @phc_clock_info: Registration structure for phc device
253 * @pps_work: pps work task for handling pps events
254 * @pps_workwq: pps work queue
255 * @nic_ts_enabled: Flag indicating if NIC generated TS events are handled
256 * @txbuf: Buffer for use when transmitting (PTP) packets to MC (avoids
257 * allocations in main data path).
258 * @timeset: Last set of synchronisation statistics.
260 struct efx_ptp_data
{
262 struct efx_channel
*channel
;
264 struct sk_buff_head rxq
;
265 struct sk_buff_head txq
;
266 struct list_head evt_list
;
267 struct list_head evt_free_list
;
270 struct efx_ptp_event_rx rx_evts
[MAX_RECEIVE_EVENTS
];
271 struct workqueue_struct
*workwq
;
272 struct work_struct work
;
275 u32 rxfilter_general
;
276 bool rxfilter_installed
;
277 struct hwtstamp_config config
;
280 unsigned int time_format
;
281 void (*ns_to_nic_time
)(s64 ns
, u32
*nic_major
, u32
*nic_minor
);
282 ktime_t (*nic_to_kernel_time
)(u32 nic_major
, u32 nic_minor
,
284 unsigned int min_synchronisation_ns
;
291 efx_qword_t evt_frags
[MAX_EVENT_FRAGS
];
294 struct efx_buffer start
;
295 struct pps_event_time host_time_pps
;
297 struct ptp_clock
*phc_clock
;
298 struct ptp_clock_info phc_clock_info
;
299 struct work_struct pps_work
;
300 struct workqueue_struct
*pps_workwq
;
302 MCDI_DECLARE_BUF(txbuf
, MC_CMD_PTP_IN_TRANSMIT_LENMAX
);
303 struct efx_ptp_timeset
304 timeset
[MC_CMD_PTP_OUT_SYNCHRONIZE_TIMESET_MAXNUM
];
307 static int efx_phc_adjfreq(struct ptp_clock_info
*ptp
, s32 delta
);
308 static int efx_phc_adjtime(struct ptp_clock_info
*ptp
, s64 delta
);
309 static int efx_phc_gettime(struct ptp_clock_info
*ptp
, struct timespec
*ts
);
310 static int efx_phc_settime(struct ptp_clock_info
*ptp
,
311 const struct timespec
*e_ts
);
312 static int efx_phc_enable(struct ptp_clock_info
*ptp
,
313 struct ptp_clock_request
*request
, int on
);
315 /* For Siena platforms NIC time is s and ns */
316 static void efx_ptp_ns_to_s_ns(s64 ns
, u32
*nic_major
, u32
*nic_minor
)
318 struct timespec ts
= ns_to_timespec(ns
);
319 *nic_major
= ts
.tv_sec
;
320 *nic_minor
= ts
.tv_nsec
;
323 static ktime_t
efx_ptp_s_ns_to_ktime_correction(u32 nic_major
, u32 nic_minor
,
326 ktime_t kt
= ktime_set(nic_major
, nic_minor
);
328 kt
= ktime_add_ns(kt
, (u64
)correction
);
330 kt
= ktime_sub_ns(kt
, (u64
)-correction
);
334 /* To convert from s27 format to ns we multiply then divide by a power of 2.
335 * For the conversion from ns to s27, the operation is also converted to a
336 * multiply and shift.
338 #define S27_TO_NS_SHIFT (27)
339 #define NS_TO_S27_MULT (((1ULL << 63) + NSEC_PER_SEC / 2) / NSEC_PER_SEC)
340 #define NS_TO_S27_SHIFT (63 - S27_TO_NS_SHIFT)
341 #define S27_MINOR_MAX (1 << S27_TO_NS_SHIFT)
343 /* For Huntington platforms NIC time is in seconds and fractions of a second
344 * where the minor register only uses 27 bits in units of 2^-27s.
346 static void efx_ptp_ns_to_s27(s64 ns
, u32
*nic_major
, u32
*nic_minor
)
348 struct timespec ts
= ns_to_timespec(ns
);
350 u32 min
= (u32
)(((u64
)ts
.tv_nsec
* NS_TO_S27_MULT
+
351 (1ULL << (NS_TO_S27_SHIFT
- 1))) >> NS_TO_S27_SHIFT
);
353 /* The conversion can result in the minor value exceeding the maximum.
354 * In this case, round up to the next second.
356 if (min
>= S27_MINOR_MAX
) {
357 min
-= S27_MINOR_MAX
;
365 static inline ktime_t
efx_ptp_s27_to_ktime(u32 nic_major
, u32 nic_minor
)
367 u32 ns
= (u32
)(((u64
)nic_minor
* NSEC_PER_SEC
+
368 (1ULL << (S27_TO_NS_SHIFT
- 1))) >> S27_TO_NS_SHIFT
);
369 return ktime_set(nic_major
, ns
);
372 static ktime_t
efx_ptp_s27_to_ktime_correction(u32 nic_major
, u32 nic_minor
,
375 /* Apply the correction and deal with carry */
376 nic_minor
+= correction
;
377 if ((s32
)nic_minor
< 0) {
378 nic_minor
+= S27_MINOR_MAX
;
380 } else if (nic_minor
>= S27_MINOR_MAX
) {
381 nic_minor
-= S27_MINOR_MAX
;
385 return efx_ptp_s27_to_ktime(nic_major
, nic_minor
);
388 /* Get PTP attributes and set up time conversions */
389 static int efx_ptp_get_attributes(struct efx_nic
*efx
)
391 MCDI_DECLARE_BUF(inbuf
, MC_CMD_PTP_IN_GET_ATTRIBUTES_LEN
);
392 MCDI_DECLARE_BUF(outbuf
, MC_CMD_PTP_OUT_GET_ATTRIBUTES_LEN
);
393 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
398 /* Get the PTP attributes. If the NIC doesn't support the operation we
399 * use the default format for compatibility with older NICs i.e.
400 * seconds and nanoseconds.
402 MCDI_SET_DWORD(inbuf
, PTP_IN_OP
, MC_CMD_PTP_OP_GET_ATTRIBUTES
);
403 MCDI_SET_DWORD(inbuf
, PTP_IN_PERIPH_ID
, 0);
404 rc
= efx_mcdi_rpc(efx
, MC_CMD_PTP
, inbuf
, sizeof(inbuf
),
405 outbuf
, sizeof(outbuf
), &out_len
);
407 fmt
= MCDI_DWORD(outbuf
, PTP_OUT_GET_ATTRIBUTES_TIME_FORMAT
);
408 else if (rc
== -EINVAL
)
409 fmt
= MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_NANOSECONDS
;
413 if (fmt
== MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_27FRACTION
) {
414 ptp
->ns_to_nic_time
= efx_ptp_ns_to_s27
;
415 ptp
->nic_to_kernel_time
= efx_ptp_s27_to_ktime_correction
;
416 } else if (fmt
== MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_NANOSECONDS
) {
417 ptp
->ns_to_nic_time
= efx_ptp_ns_to_s_ns
;
418 ptp
->nic_to_kernel_time
= efx_ptp_s_ns_to_ktime_correction
;
423 ptp
->time_format
= fmt
;
425 /* MC_CMD_PTP_OP_GET_ATTRIBUTES is an extended version of an older
426 * operation MC_CMD_PTP_OP_GET_TIME_FORMAT that also returns a value
427 * to use for the minimum acceptable corrected synchronization window.
428 * If we have the extra information store it. For older firmware that
429 * does not implement the extended command use the default value.
431 if (rc
== 0 && out_len
>= MC_CMD_PTP_OUT_GET_ATTRIBUTES_LEN
)
432 ptp
->min_synchronisation_ns
=
434 PTP_OUT_GET_ATTRIBUTES_SYNC_WINDOW_MIN
);
436 ptp
->min_synchronisation_ns
= DEFAULT_MIN_SYNCHRONISATION_NS
;
441 /* Get PTP timestamp corrections */
442 static int efx_ptp_get_timestamp_corrections(struct efx_nic
*efx
)
444 MCDI_DECLARE_BUF(inbuf
, MC_CMD_PTP_IN_GET_TIMESTAMP_CORRECTIONS_LEN
);
445 MCDI_DECLARE_BUF(outbuf
, MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_LEN
);
448 /* Get the timestamp corrections from the NIC. If this operation is
449 * not supported (older NICs) then no correction is required.
451 MCDI_SET_DWORD(inbuf
, PTP_IN_OP
,
452 MC_CMD_PTP_OP_GET_TIMESTAMP_CORRECTIONS
);
453 MCDI_SET_DWORD(inbuf
, PTP_IN_PERIPH_ID
, 0);
455 rc
= efx_mcdi_rpc(efx
, MC_CMD_PTP
, inbuf
, sizeof(inbuf
),
456 outbuf
, sizeof(outbuf
), NULL
);
458 efx
->ptp_data
->ts_corrections
.tx
= MCDI_DWORD(outbuf
,
459 PTP_OUT_GET_TIMESTAMP_CORRECTIONS_TRANSMIT
);
460 efx
->ptp_data
->ts_corrections
.rx
= MCDI_DWORD(outbuf
,
461 PTP_OUT_GET_TIMESTAMP_CORRECTIONS_RECEIVE
);
462 efx
->ptp_data
->ts_corrections
.pps_out
= MCDI_DWORD(outbuf
,
463 PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_OUT
);
464 efx
->ptp_data
->ts_corrections
.pps_in
= MCDI_DWORD(outbuf
,
465 PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_IN
);
466 } else if (rc
== -EINVAL
) {
467 efx
->ptp_data
->ts_corrections
.tx
= 0;
468 efx
->ptp_data
->ts_corrections
.rx
= 0;
469 efx
->ptp_data
->ts_corrections
.pps_out
= 0;
470 efx
->ptp_data
->ts_corrections
.pps_in
= 0;
478 /* Enable MCDI PTP support. */
479 static int efx_ptp_enable(struct efx_nic
*efx
)
481 MCDI_DECLARE_BUF(inbuf
, MC_CMD_PTP_IN_ENABLE_LEN
);
482 MCDI_DECLARE_BUF_OUT_OR_ERR(outbuf
, 0);
485 MCDI_SET_DWORD(inbuf
, PTP_IN_OP
, MC_CMD_PTP_OP_ENABLE
);
486 MCDI_SET_DWORD(inbuf
, PTP_IN_PERIPH_ID
, 0);
487 MCDI_SET_DWORD(inbuf
, PTP_IN_ENABLE_QUEUE
,
488 efx
->ptp_data
->channel
?
489 efx
->ptp_data
->channel
->channel
: 0);
490 MCDI_SET_DWORD(inbuf
, PTP_IN_ENABLE_MODE
, efx
->ptp_data
->mode
);
492 rc
= efx_mcdi_rpc_quiet(efx
, MC_CMD_PTP
, inbuf
, sizeof(inbuf
),
493 outbuf
, sizeof(outbuf
), NULL
);
494 rc
= (rc
== -EALREADY
) ? 0 : rc
;
496 efx_mcdi_display_error(efx
, MC_CMD_PTP
,
497 MC_CMD_PTP_IN_ENABLE_LEN
,
498 outbuf
, sizeof(outbuf
), rc
);
502 /* Disable MCDI PTP support.
504 * Note that this function should never rely on the presence of ptp_data -
505 * may be called before that exists.
507 static int efx_ptp_disable(struct efx_nic
*efx
)
509 MCDI_DECLARE_BUF(inbuf
, MC_CMD_PTP_IN_DISABLE_LEN
);
510 MCDI_DECLARE_BUF_OUT_OR_ERR(outbuf
, 0);
513 MCDI_SET_DWORD(inbuf
, PTP_IN_OP
, MC_CMD_PTP_OP_DISABLE
);
514 MCDI_SET_DWORD(inbuf
, PTP_IN_PERIPH_ID
, 0);
515 rc
= efx_mcdi_rpc_quiet(efx
, MC_CMD_PTP
, inbuf
, sizeof(inbuf
),
516 outbuf
, sizeof(outbuf
), NULL
);
517 rc
= (rc
== -EALREADY
) ? 0 : rc
;
519 efx_mcdi_display_error(efx
, MC_CMD_PTP
,
520 MC_CMD_PTP_IN_DISABLE_LEN
,
521 outbuf
, sizeof(outbuf
), rc
);
525 static void efx_ptp_deliver_rx_queue(struct sk_buff_head
*q
)
529 while ((skb
= skb_dequeue(q
))) {
531 netif_receive_skb(skb
);
536 static void efx_ptp_handle_no_channel(struct efx_nic
*efx
)
538 netif_err(efx
, drv
, efx
->net_dev
,
539 "ERROR: PTP requires MSI-X and 1 additional interrupt"
540 "vector. PTP disabled\n");
543 /* Repeatedly send the host time to the MC which will capture the hardware
546 static void efx_ptp_send_times(struct efx_nic
*efx
,
547 struct pps_event_time
*last_time
)
549 struct pps_event_time now
;
550 struct timespec limit
;
551 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
552 struct timespec start
;
553 int *mc_running
= ptp
->start
.addr
;
558 timespec_add_ns(&limit
, SYNCHRONISE_PERIOD_NS
);
560 /* Write host time for specified period or until MC is done */
561 while ((timespec_compare(&now
.ts_real
, &limit
) < 0) &&
562 ACCESS_ONCE(*mc_running
)) {
563 struct timespec update_time
;
564 unsigned int host_time
;
566 /* Don't update continuously to avoid saturating the PCIe bus */
567 update_time
= now
.ts_real
;
568 timespec_add_ns(&update_time
, SYNCHRONISATION_GRANULARITY_NS
);
571 } while ((timespec_compare(&now
.ts_real
, &update_time
) < 0) &&
572 ACCESS_ONCE(*mc_running
));
574 /* Synchronise NIC with single word of time only */
575 host_time
= (now
.ts_real
.tv_sec
<< MC_NANOSECOND_BITS
|
576 now
.ts_real
.tv_nsec
);
577 /* Update host time in NIC memory */
578 efx
->type
->ptp_write_host_time(efx
, host_time
);
583 /* Read a timeset from the MC's results and partial process. */
584 static void efx_ptp_read_timeset(MCDI_DECLARE_STRUCT_PTR(data
),
585 struct efx_ptp_timeset
*timeset
)
587 unsigned start_ns
, end_ns
;
589 timeset
->host_start
= MCDI_DWORD(data
, PTP_OUT_SYNCHRONIZE_HOSTSTART
);
590 timeset
->major
= MCDI_DWORD(data
, PTP_OUT_SYNCHRONIZE_MAJOR
);
591 timeset
->minor
= MCDI_DWORD(data
, PTP_OUT_SYNCHRONIZE_MINOR
);
592 timeset
->host_end
= MCDI_DWORD(data
, PTP_OUT_SYNCHRONIZE_HOSTEND
),
593 timeset
->wait
= MCDI_DWORD(data
, PTP_OUT_SYNCHRONIZE_WAITNS
);
596 start_ns
= timeset
->host_start
& MC_NANOSECOND_MASK
;
597 end_ns
= timeset
->host_end
& MC_NANOSECOND_MASK
;
598 /* Allow for rollover */
599 if (end_ns
< start_ns
)
600 end_ns
+= NSEC_PER_SEC
;
601 /* Determine duration of operation */
602 timeset
->window
= end_ns
- start_ns
;
605 /* Process times received from MC.
607 * Extract times from returned results, and establish the minimum value
608 * seen. The minimum value represents the "best" possible time and events
609 * too much greater than this are rejected - the machine is, perhaps, too
610 * busy. A number of readings are taken so that, hopefully, at least one good
611 * synchronisation will be seen in the results.
614 efx_ptp_process_times(struct efx_nic
*efx
, MCDI_DECLARE_STRUCT_PTR(synch_buf
),
615 size_t response_length
,
616 const struct pps_event_time
*last_time
)
618 unsigned number_readings
=
619 MCDI_VAR_ARRAY_LEN(response_length
,
620 PTP_OUT_SYNCHRONIZE_TIMESET
);
623 unsigned last_good
= 0;
624 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
627 struct timespec delta
;
630 if (number_readings
== 0)
633 /* Read the set of results and find the last good host-MC
634 * synchronization result. The MC times when it finishes reading the
635 * host time so the corrected window time should be fairly constant
636 * for a given platform.
638 for (i
= 0; i
< number_readings
; i
++) {
639 s32 window
, corrected
;
640 struct timespec wait
;
642 efx_ptp_read_timeset(
643 MCDI_ARRAY_STRUCT_PTR(synch_buf
,
644 PTP_OUT_SYNCHRONIZE_TIMESET
, i
),
647 wait
= ktime_to_timespec(
648 ptp
->nic_to_kernel_time(0, ptp
->timeset
[i
].wait
, 0));
649 window
= ptp
->timeset
[i
].window
;
650 corrected
= window
- wait
.tv_nsec
;
652 /* We expect the uncorrected synchronization window to be at
653 * least as large as the interval between host start and end
654 * times. If it is smaller than this then this is mostly likely
655 * to be a consequence of the host's time being adjusted.
656 * Check that the corrected sync window is in a reasonable
657 * range. If it is out of range it is likely to be because an
658 * interrupt or other delay occurred between reading the system
659 * time and writing it to MC memory.
661 if (window
>= SYNCHRONISATION_GRANULARITY_NS
&&
662 corrected
< MAX_SYNCHRONISATION_NS
&&
663 corrected
>= ptp
->min_synchronisation_ns
) {
670 netif_warn(efx
, drv
, efx
->net_dev
,
671 "PTP no suitable synchronisations\n");
675 /* Convert the NIC time into kernel time. No correction is required-
676 * this time is the output of a firmware process.
678 mc_time
= ptp
->nic_to_kernel_time(ptp
->timeset
[last_good
].major
,
679 ptp
->timeset
[last_good
].minor
, 0);
681 /* Calculate delay from actual PPS to last_time */
682 delta
= ktime_to_timespec(mc_time
);
684 last_time
->ts_real
.tv_nsec
-
685 (ptp
->timeset
[last_good
].host_start
& MC_NANOSECOND_MASK
);
687 /* It is possible that the seconds rolled over between taking
688 * the start reading and the last value written by the host. The
689 * timescales are such that a gap of more than one second is never
692 start_sec
= ptp
->timeset
[last_good
].host_start
>> MC_NANOSECOND_BITS
;
693 last_sec
= last_time
->ts_real
.tv_sec
& MC_SECOND_MASK
;
694 if (start_sec
!= last_sec
) {
695 if (((start_sec
+ 1) & MC_SECOND_MASK
) != last_sec
) {
696 netif_warn(efx
, hw
, efx
->net_dev
,
697 "PTP bad synchronisation seconds\n");
706 ptp
->host_time_pps
= *last_time
;
707 pps_sub_ts(&ptp
->host_time_pps
, delta
);
712 /* Synchronize times between the host and the MC */
713 static int efx_ptp_synchronize(struct efx_nic
*efx
, unsigned int num_readings
)
715 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
716 MCDI_DECLARE_BUF(synch_buf
, MC_CMD_PTP_OUT_SYNCHRONIZE_LENMAX
);
717 size_t response_length
;
719 unsigned long timeout
;
720 struct pps_event_time last_time
= {};
721 unsigned int loops
= 0;
722 int *start
= ptp
->start
.addr
;
724 MCDI_SET_DWORD(synch_buf
, PTP_IN_OP
, MC_CMD_PTP_OP_SYNCHRONIZE
);
725 MCDI_SET_DWORD(synch_buf
, PTP_IN_PERIPH_ID
, 0);
726 MCDI_SET_DWORD(synch_buf
, PTP_IN_SYNCHRONIZE_NUMTIMESETS
,
728 MCDI_SET_QWORD(synch_buf
, PTP_IN_SYNCHRONIZE_START_ADDR
,
729 ptp
->start
.dma_addr
);
731 /* Clear flag that signals MC ready */
732 ACCESS_ONCE(*start
) = 0;
733 rc
= efx_mcdi_rpc_start(efx
, MC_CMD_PTP
, synch_buf
,
734 MC_CMD_PTP_IN_SYNCHRONIZE_LEN
);
735 EFX_BUG_ON_PARANOID(rc
);
737 /* Wait for start from MCDI (or timeout) */
738 timeout
= jiffies
+ msecs_to_jiffies(MAX_SYNCHRONISE_WAIT_MS
);
739 while (!ACCESS_ONCE(*start
) && (time_before(jiffies
, timeout
))) {
740 udelay(20); /* Usually start MCDI execution quickly */
744 if (ACCESS_ONCE(*start
))
745 efx_ptp_send_times(efx
, &last_time
);
747 /* Collect results */
748 rc
= efx_mcdi_rpc_finish(efx
, MC_CMD_PTP
,
749 MC_CMD_PTP_IN_SYNCHRONIZE_LEN
,
750 synch_buf
, sizeof(synch_buf
),
753 rc
= efx_ptp_process_times(efx
, synch_buf
, response_length
,
759 /* Transmit a PTP packet, via the MCDI interface, to the wire. */
760 static int efx_ptp_xmit_skb(struct efx_nic
*efx
, struct sk_buff
*skb
)
762 struct efx_ptp_data
*ptp_data
= efx
->ptp_data
;
763 struct skb_shared_hwtstamps timestamps
;
765 MCDI_DECLARE_BUF(txtime
, MC_CMD_PTP_OUT_TRANSMIT_LEN
);
768 MCDI_SET_DWORD(ptp_data
->txbuf
, PTP_IN_OP
, MC_CMD_PTP_OP_TRANSMIT
);
769 MCDI_SET_DWORD(ptp_data
->txbuf
, PTP_IN_PERIPH_ID
, 0);
770 MCDI_SET_DWORD(ptp_data
->txbuf
, PTP_IN_TRANSMIT_LENGTH
, skb
->len
);
771 if (skb_shinfo(skb
)->nr_frags
!= 0) {
772 rc
= skb_linearize(skb
);
777 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
778 rc
= skb_checksum_help(skb
);
782 skb_copy_from_linear_data(skb
,
783 MCDI_PTR(ptp_data
->txbuf
,
784 PTP_IN_TRANSMIT_PACKET
),
786 rc
= efx_mcdi_rpc(efx
, MC_CMD_PTP
,
787 ptp_data
->txbuf
, MC_CMD_PTP_IN_TRANSMIT_LEN(skb
->len
),
788 txtime
, sizeof(txtime
), &len
);
792 memset(×tamps
, 0, sizeof(timestamps
));
793 timestamps
.hwtstamp
= ptp_data
->nic_to_kernel_time(
794 MCDI_DWORD(txtime
, PTP_OUT_TRANSMIT_MAJOR
),
795 MCDI_DWORD(txtime
, PTP_OUT_TRANSMIT_MINOR
),
796 ptp_data
->ts_corrections
.tx
);
798 skb_tstamp_tx(skb
, ×tamps
);
808 static void efx_ptp_drop_time_expired_events(struct efx_nic
*efx
)
810 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
811 struct list_head
*cursor
;
812 struct list_head
*next
;
814 if (ptp
->rx_ts_inline
)
817 /* Drop time-expired events */
818 spin_lock_bh(&ptp
->evt_lock
);
819 if (!list_empty(&ptp
->evt_list
)) {
820 list_for_each_safe(cursor
, next
, &ptp
->evt_list
) {
821 struct efx_ptp_event_rx
*evt
;
823 evt
= list_entry(cursor
, struct efx_ptp_event_rx
,
825 if (time_after(jiffies
, evt
->expiry
)) {
826 list_move(&evt
->link
, &ptp
->evt_free_list
);
827 netif_warn(efx
, hw
, efx
->net_dev
,
828 "PTP rx event dropped\n");
832 /* If the event overflow flag is set and the event list is now empty
833 * clear the flag to re-enable the overflow warning message.
835 if (ptp
->evt_overflow
&& list_empty(&ptp
->evt_list
))
836 ptp
->evt_overflow
= false;
837 spin_unlock_bh(&ptp
->evt_lock
);
840 static enum ptp_packet_state
efx_ptp_match_rx(struct efx_nic
*efx
,
843 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
845 struct list_head
*cursor
;
846 struct list_head
*next
;
847 struct efx_ptp_match
*match
;
848 enum ptp_packet_state rc
= PTP_PACKET_STATE_UNMATCHED
;
850 WARN_ON_ONCE(ptp
->rx_ts_inline
);
852 spin_lock_bh(&ptp
->evt_lock
);
853 evts_waiting
= !list_empty(&ptp
->evt_list
);
854 spin_unlock_bh(&ptp
->evt_lock
);
857 return PTP_PACKET_STATE_UNMATCHED
;
859 match
= (struct efx_ptp_match
*)skb
->cb
;
860 /* Look for a matching timestamp in the event queue */
861 spin_lock_bh(&ptp
->evt_lock
);
862 list_for_each_safe(cursor
, next
, &ptp
->evt_list
) {
863 struct efx_ptp_event_rx
*evt
;
865 evt
= list_entry(cursor
, struct efx_ptp_event_rx
, link
);
866 if ((evt
->seq0
== match
->words
[0]) &&
867 (evt
->seq1
== match
->words
[1])) {
868 struct skb_shared_hwtstamps
*timestamps
;
870 /* Match - add in hardware timestamp */
871 timestamps
= skb_hwtstamps(skb
);
872 timestamps
->hwtstamp
= evt
->hwtimestamp
;
874 match
->state
= PTP_PACKET_STATE_MATCHED
;
875 rc
= PTP_PACKET_STATE_MATCHED
;
876 list_move(&evt
->link
, &ptp
->evt_free_list
);
880 /* If the event overflow flag is set and the event list is now empty
881 * clear the flag to re-enable the overflow warning message.
883 if (ptp
->evt_overflow
&& list_empty(&ptp
->evt_list
))
884 ptp
->evt_overflow
= false;
885 spin_unlock_bh(&ptp
->evt_lock
);
890 /* Process any queued receive events and corresponding packets
892 * q is returned with all the packets that are ready for delivery.
893 * true is returned if at least one of those packets requires
896 static bool efx_ptp_process_events(struct efx_nic
*efx
, struct sk_buff_head
*q
)
898 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
902 while ((skb
= skb_dequeue(&ptp
->rxq
))) {
903 struct efx_ptp_match
*match
;
905 match
= (struct efx_ptp_match
*)skb
->cb
;
906 if (match
->state
== PTP_PACKET_STATE_MATCH_UNWANTED
) {
907 __skb_queue_tail(q
, skb
);
908 } else if (efx_ptp_match_rx(efx
, skb
) ==
909 PTP_PACKET_STATE_MATCHED
) {
911 __skb_queue_tail(q
, skb
);
912 } else if (time_after(jiffies
, match
->expiry
)) {
913 match
->state
= PTP_PACKET_STATE_TIMED_OUT
;
915 netif_warn(efx
, rx_err
, efx
->net_dev
,
916 "PTP packet - no timestamp seen\n");
917 __skb_queue_tail(q
, skb
);
919 /* Replace unprocessed entry and stop */
920 skb_queue_head(&ptp
->rxq
, skb
);
928 /* Complete processing of a received packet */
929 static inline void efx_ptp_process_rx(struct efx_nic
*efx
, struct sk_buff
*skb
)
932 netif_receive_skb(skb
);
936 static void efx_ptp_remove_multicast_filters(struct efx_nic
*efx
)
938 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
940 if (ptp
->rxfilter_installed
) {
941 efx_filter_remove_id_safe(efx
, EFX_FILTER_PRI_REQUIRED
,
942 ptp
->rxfilter_general
);
943 efx_filter_remove_id_safe(efx
, EFX_FILTER_PRI_REQUIRED
,
944 ptp
->rxfilter_event
);
945 ptp
->rxfilter_installed
= false;
949 static int efx_ptp_insert_multicast_filters(struct efx_nic
*efx
)
951 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
952 struct efx_filter_spec rxfilter
;
955 if (!ptp
->channel
|| ptp
->rxfilter_installed
)
958 /* Must filter on both event and general ports to ensure
959 * that there is no packet re-ordering.
961 efx_filter_init_rx(&rxfilter
, EFX_FILTER_PRI_REQUIRED
, 0,
963 efx_channel_get_rx_queue(ptp
->channel
)));
964 rc
= efx_filter_set_ipv4_local(&rxfilter
, IPPROTO_UDP
,
966 htons(PTP_EVENT_PORT
));
970 rc
= efx_filter_insert_filter(efx
, &rxfilter
, true);
973 ptp
->rxfilter_event
= rc
;
975 efx_filter_init_rx(&rxfilter
, EFX_FILTER_PRI_REQUIRED
, 0,
977 efx_channel_get_rx_queue(ptp
->channel
)));
978 rc
= efx_filter_set_ipv4_local(&rxfilter
, IPPROTO_UDP
,
980 htons(PTP_GENERAL_PORT
));
984 rc
= efx_filter_insert_filter(efx
, &rxfilter
, true);
987 ptp
->rxfilter_general
= rc
;
989 ptp
->rxfilter_installed
= true;
993 efx_filter_remove_id_safe(efx
, EFX_FILTER_PRI_REQUIRED
,
994 ptp
->rxfilter_event
);
998 static int efx_ptp_start(struct efx_nic
*efx
)
1000 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
1003 ptp
->reset_required
= false;
1005 rc
= efx_ptp_insert_multicast_filters(efx
);
1009 rc
= efx_ptp_enable(efx
);
1013 ptp
->evt_frag_idx
= 0;
1014 ptp
->current_adjfreq
= 0;
1019 efx_ptp_remove_multicast_filters(efx
);
1023 static int efx_ptp_stop(struct efx_nic
*efx
)
1025 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
1026 struct list_head
*cursor
;
1027 struct list_head
*next
;
1033 rc
= efx_ptp_disable(efx
);
1035 efx_ptp_remove_multicast_filters(efx
);
1037 /* Make sure RX packets are really delivered */
1038 efx_ptp_deliver_rx_queue(&efx
->ptp_data
->rxq
);
1039 skb_queue_purge(&efx
->ptp_data
->txq
);
1041 /* Drop any pending receive events */
1042 spin_lock_bh(&efx
->ptp_data
->evt_lock
);
1043 list_for_each_safe(cursor
, next
, &efx
->ptp_data
->evt_list
) {
1044 list_move(cursor
, &efx
->ptp_data
->evt_free_list
);
1046 ptp
->evt_overflow
= false;
1047 spin_unlock_bh(&efx
->ptp_data
->evt_lock
);
1052 static int efx_ptp_restart(struct efx_nic
*efx
)
1054 if (efx
->ptp_data
&& efx
->ptp_data
->enabled
)
1055 return efx_ptp_start(efx
);
1059 static void efx_ptp_pps_worker(struct work_struct
*work
)
1061 struct efx_ptp_data
*ptp
=
1062 container_of(work
, struct efx_ptp_data
, pps_work
);
1063 struct efx_nic
*efx
= ptp
->efx
;
1064 struct ptp_clock_event ptp_evt
;
1066 if (efx_ptp_synchronize(efx
, PTP_SYNC_ATTEMPTS
))
1069 ptp_evt
.type
= PTP_CLOCK_PPSUSR
;
1070 ptp_evt
.pps_times
= ptp
->host_time_pps
;
1071 ptp_clock_event(ptp
->phc_clock
, &ptp_evt
);
1074 static void efx_ptp_worker(struct work_struct
*work
)
1076 struct efx_ptp_data
*ptp_data
=
1077 container_of(work
, struct efx_ptp_data
, work
);
1078 struct efx_nic
*efx
= ptp_data
->efx
;
1079 struct sk_buff
*skb
;
1080 struct sk_buff_head tempq
;
1082 if (ptp_data
->reset_required
) {
1088 efx_ptp_drop_time_expired_events(efx
);
1090 __skb_queue_head_init(&tempq
);
1091 if (efx_ptp_process_events(efx
, &tempq
) ||
1092 !skb_queue_empty(&ptp_data
->txq
)) {
1094 while ((skb
= skb_dequeue(&ptp_data
->txq
)))
1095 efx_ptp_xmit_skb(efx
, skb
);
1098 while ((skb
= __skb_dequeue(&tempq
)))
1099 efx_ptp_process_rx(efx
, skb
);
1102 static const struct ptp_clock_info efx_phc_clock_info
= {
1103 .owner
= THIS_MODULE
,
1110 .adjfreq
= efx_phc_adjfreq
,
1111 .adjtime
= efx_phc_adjtime
,
1112 .gettime
= efx_phc_gettime
,
1113 .settime
= efx_phc_settime
,
1114 .enable
= efx_phc_enable
,
1117 /* Initialise PTP state. */
1118 int efx_ptp_probe(struct efx_nic
*efx
, struct efx_channel
*channel
)
1120 struct efx_ptp_data
*ptp
;
1124 ptp
= kzalloc(sizeof(struct efx_ptp_data
), GFP_KERNEL
);
1125 efx
->ptp_data
= ptp
;
1130 ptp
->channel
= channel
;
1131 ptp
->rx_ts_inline
= efx_nic_rev(efx
) >= EFX_REV_HUNT_A0
;
1133 rc
= efx_nic_alloc_buffer(efx
, &ptp
->start
, sizeof(int), GFP_KERNEL
);
1137 skb_queue_head_init(&ptp
->rxq
);
1138 skb_queue_head_init(&ptp
->txq
);
1139 ptp
->workwq
= create_singlethread_workqueue("sfc_ptp");
1145 INIT_WORK(&ptp
->work
, efx_ptp_worker
);
1146 ptp
->config
.flags
= 0;
1147 ptp
->config
.tx_type
= HWTSTAMP_TX_OFF
;
1148 ptp
->config
.rx_filter
= HWTSTAMP_FILTER_NONE
;
1149 INIT_LIST_HEAD(&ptp
->evt_list
);
1150 INIT_LIST_HEAD(&ptp
->evt_free_list
);
1151 spin_lock_init(&ptp
->evt_lock
);
1152 for (pos
= 0; pos
< MAX_RECEIVE_EVENTS
; pos
++)
1153 list_add(&ptp
->rx_evts
[pos
].link
, &ptp
->evt_free_list
);
1154 ptp
->evt_overflow
= false;
1156 /* Get the NIC PTP attributes and set up time conversions */
1157 rc
= efx_ptp_get_attributes(efx
);
1161 /* Get the timestamp corrections */
1162 rc
= efx_ptp_get_timestamp_corrections(efx
);
1166 ptp
->phc_clock_info
= efx_phc_clock_info
;
1167 ptp
->phc_clock
= ptp_clock_register(&ptp
->phc_clock_info
,
1168 &efx
->pci_dev
->dev
);
1169 if (IS_ERR(ptp
->phc_clock
)) {
1170 rc
= PTR_ERR(ptp
->phc_clock
);
1174 INIT_WORK(&ptp
->pps_work
, efx_ptp_pps_worker
);
1175 ptp
->pps_workwq
= create_singlethread_workqueue("sfc_pps");
1176 if (!ptp
->pps_workwq
) {
1180 ptp
->nic_ts_enabled
= false;
1184 ptp_clock_unregister(efx
->ptp_data
->phc_clock
);
1187 destroy_workqueue(efx
->ptp_data
->workwq
);
1190 efx_nic_free_buffer(efx
, &ptp
->start
);
1193 kfree(efx
->ptp_data
);
1194 efx
->ptp_data
= NULL
;
1199 /* Initialise PTP channel.
1201 * Setting core_index to zero causes the queue to be initialised and doesn't
1202 * overlap with 'rxq0' because ptp.c doesn't use skb_record_rx_queue.
1204 static int efx_ptp_probe_channel(struct efx_channel
*channel
)
1206 struct efx_nic
*efx
= channel
->efx
;
1208 channel
->irq_moderation
= 0;
1209 channel
->rx_queue
.core_index
= 0;
1211 return efx_ptp_probe(efx
, channel
);
1214 void efx_ptp_remove(struct efx_nic
*efx
)
1219 (void)efx_ptp_disable(efx
);
1221 cancel_work_sync(&efx
->ptp_data
->work
);
1222 cancel_work_sync(&efx
->ptp_data
->pps_work
);
1224 skb_queue_purge(&efx
->ptp_data
->rxq
);
1225 skb_queue_purge(&efx
->ptp_data
->txq
);
1227 ptp_clock_unregister(efx
->ptp_data
->phc_clock
);
1229 destroy_workqueue(efx
->ptp_data
->workwq
);
1230 destroy_workqueue(efx
->ptp_data
->pps_workwq
);
1232 efx_nic_free_buffer(efx
, &efx
->ptp_data
->start
);
1233 kfree(efx
->ptp_data
);
1236 static void efx_ptp_remove_channel(struct efx_channel
*channel
)
1238 efx_ptp_remove(channel
->efx
);
1241 static void efx_ptp_get_channel_name(struct efx_channel
*channel
,
1242 char *buf
, size_t len
)
1244 snprintf(buf
, len
, "%s-ptp", channel
->efx
->name
);
1247 /* Determine whether this packet should be processed by the PTP module
1248 * or transmitted conventionally.
1250 bool efx_ptp_is_ptp_tx(struct efx_nic
*efx
, struct sk_buff
*skb
)
1252 return efx
->ptp_data
&&
1253 efx
->ptp_data
->enabled
&&
1254 skb
->len
>= PTP_MIN_LENGTH
&&
1255 skb
->len
<= MC_CMD_PTP_IN_TRANSMIT_PACKET_MAXNUM
&&
1256 likely(skb
->protocol
== htons(ETH_P_IP
)) &&
1257 skb_transport_header_was_set(skb
) &&
1258 skb_network_header_len(skb
) >= sizeof(struct iphdr
) &&
1259 ip_hdr(skb
)->protocol
== IPPROTO_UDP
&&
1261 skb_transport_offset(skb
) + sizeof(struct udphdr
) &&
1262 udp_hdr(skb
)->dest
== htons(PTP_EVENT_PORT
);
1265 /* Receive a PTP packet. Packets are queued until the arrival of
1266 * the receive timestamp from the MC - this will probably occur after the
1267 * packet arrival because of the processing in the MC.
1269 static bool efx_ptp_rx(struct efx_channel
*channel
, struct sk_buff
*skb
)
1271 struct efx_nic
*efx
= channel
->efx
;
1272 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
1273 struct efx_ptp_match
*match
= (struct efx_ptp_match
*)skb
->cb
;
1274 u8
*match_data_012
, *match_data_345
;
1275 unsigned int version
;
1277 match
->expiry
= jiffies
+ msecs_to_jiffies(PKT_EVENT_LIFETIME_MS
);
1279 /* Correct version? */
1280 if (ptp
->mode
== MC_CMD_PTP_MODE_V1
) {
1281 if (!pskb_may_pull(skb
, PTP_V1_MIN_LENGTH
)) {
1284 version
= ntohs(*(__be16
*)&skb
->data
[PTP_V1_VERSION_OFFSET
]);
1285 if (version
!= PTP_VERSION_V1
) {
1289 /* PTP V1 uses all six bytes of the UUID to match the packet
1292 match_data_012
= skb
->data
+ PTP_V1_UUID_OFFSET
;
1293 match_data_345
= skb
->data
+ PTP_V1_UUID_OFFSET
+ 3;
1295 if (!pskb_may_pull(skb
, PTP_V2_MIN_LENGTH
)) {
1298 version
= skb
->data
[PTP_V2_VERSION_OFFSET
];
1299 if ((version
& PTP_VERSION_V2_MASK
) != PTP_VERSION_V2
) {
1303 /* The original V2 implementation uses bytes 2-7 of
1304 * the UUID to match the packet to the timestamp. This
1305 * discards two of the bytes of the MAC address used
1306 * to create the UUID (SF bug 33070). The PTP V2
1307 * enhanced mode fixes this issue and uses bytes 0-2
1308 * and byte 5-7 of the UUID.
1310 match_data_345
= skb
->data
+ PTP_V2_UUID_OFFSET
+ 5;
1311 if (ptp
->mode
== MC_CMD_PTP_MODE_V2
) {
1312 match_data_012
= skb
->data
+ PTP_V2_UUID_OFFSET
+ 2;
1314 match_data_012
= skb
->data
+ PTP_V2_UUID_OFFSET
+ 0;
1315 BUG_ON(ptp
->mode
!= MC_CMD_PTP_MODE_V2_ENHANCED
);
1319 /* Does this packet require timestamping? */
1320 if (ntohs(*(__be16
*)&skb
->data
[PTP_DPORT_OFFSET
]) == PTP_EVENT_PORT
) {
1321 struct skb_shared_hwtstamps
*timestamps
;
1323 match
->state
= PTP_PACKET_STATE_UNMATCHED
;
1325 /* Clear all timestamps held: filled in later */
1326 timestamps
= skb_hwtstamps(skb
);
1327 memset(timestamps
, 0, sizeof(*timestamps
));
1329 /* We expect the sequence number to be in the same position in
1330 * the packet for PTP V1 and V2
1332 BUILD_BUG_ON(PTP_V1_SEQUENCE_OFFSET
!= PTP_V2_SEQUENCE_OFFSET
);
1333 BUILD_BUG_ON(PTP_V1_SEQUENCE_LENGTH
!= PTP_V2_SEQUENCE_LENGTH
);
1335 /* Extract UUID/Sequence information */
1336 match
->words
[0] = (match_data_012
[0] |
1337 (match_data_012
[1] << 8) |
1338 (match_data_012
[2] << 16) |
1339 (match_data_345
[0] << 24));
1340 match
->words
[1] = (match_data_345
[1] |
1341 (match_data_345
[2] << 8) |
1342 (skb
->data
[PTP_V1_SEQUENCE_OFFSET
+
1343 PTP_V1_SEQUENCE_LENGTH
- 1] <<
1346 match
->state
= PTP_PACKET_STATE_MATCH_UNWANTED
;
1349 skb_queue_tail(&ptp
->rxq
, skb
);
1350 queue_work(ptp
->workwq
, &ptp
->work
);
1355 /* Transmit a PTP packet. This has to be transmitted by the MC
1356 * itself, through an MCDI call. MCDI calls aren't permitted
1357 * in the transmit path so defer the actual transmission to a suitable worker.
1359 int efx_ptp_tx(struct efx_nic
*efx
, struct sk_buff
*skb
)
1361 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
1363 skb_queue_tail(&ptp
->txq
, skb
);
1365 if ((udp_hdr(skb
)->dest
== htons(PTP_EVENT_PORT
)) &&
1366 (skb
->len
<= MC_CMD_PTP_IN_TRANSMIT_PACKET_MAXNUM
))
1367 efx_xmit_hwtstamp_pending(skb
);
1368 queue_work(ptp
->workwq
, &ptp
->work
);
1370 return NETDEV_TX_OK
;
1373 int efx_ptp_get_mode(struct efx_nic
*efx
)
1375 return efx
->ptp_data
->mode
;
1378 int efx_ptp_change_mode(struct efx_nic
*efx
, bool enable_wanted
,
1379 unsigned int new_mode
)
1381 if ((enable_wanted
!= efx
->ptp_data
->enabled
) ||
1382 (enable_wanted
&& (efx
->ptp_data
->mode
!= new_mode
))) {
1385 if (enable_wanted
) {
1386 /* Change of mode requires disable */
1387 if (efx
->ptp_data
->enabled
&&
1388 (efx
->ptp_data
->mode
!= new_mode
)) {
1389 efx
->ptp_data
->enabled
= false;
1390 rc
= efx_ptp_stop(efx
);
1395 /* Set new operating mode and establish
1396 * baseline synchronisation, which must
1399 efx
->ptp_data
->mode
= new_mode
;
1400 if (netif_running(efx
->net_dev
))
1401 rc
= efx_ptp_start(efx
);
1403 rc
= efx_ptp_synchronize(efx
,
1404 PTP_SYNC_ATTEMPTS
* 2);
1409 rc
= efx_ptp_stop(efx
);
1415 efx
->ptp_data
->enabled
= enable_wanted
;
1421 static int efx_ptp_ts_init(struct efx_nic
*efx
, struct hwtstamp_config
*init
)
1428 if ((init
->tx_type
!= HWTSTAMP_TX_OFF
) &&
1429 (init
->tx_type
!= HWTSTAMP_TX_ON
))
1432 rc
= efx
->type
->ptp_set_ts_config(efx
, init
);
1436 efx
->ptp_data
->config
= *init
;
1440 void efx_ptp_get_ts_info(struct efx_nic
*efx
, struct ethtool_ts_info
*ts_info
)
1442 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
1447 ts_info
->so_timestamping
|= (SOF_TIMESTAMPING_TX_HARDWARE
|
1448 SOF_TIMESTAMPING_RX_HARDWARE
|
1449 SOF_TIMESTAMPING_RAW_HARDWARE
);
1450 ts_info
->phc_index
= ptp_clock_index(ptp
->phc_clock
);
1451 ts_info
->tx_types
= 1 << HWTSTAMP_TX_OFF
| 1 << HWTSTAMP_TX_ON
;
1452 ts_info
->rx_filters
= ptp
->efx
->type
->hwtstamp_filters
;
1455 int efx_ptp_set_ts_config(struct efx_nic
*efx
, struct ifreq
*ifr
)
1457 struct hwtstamp_config config
;
1460 /* Not a PTP enabled port */
1464 if (copy_from_user(&config
, ifr
->ifr_data
, sizeof(config
)))
1467 rc
= efx_ptp_ts_init(efx
, &config
);
1471 return copy_to_user(ifr
->ifr_data
, &config
, sizeof(config
))
1475 int efx_ptp_get_ts_config(struct efx_nic
*efx
, struct ifreq
*ifr
)
1480 return copy_to_user(ifr
->ifr_data
, &efx
->ptp_data
->config
,
1481 sizeof(efx
->ptp_data
->config
)) ? -EFAULT
: 0;
1484 static void ptp_event_failure(struct efx_nic
*efx
, int expected_frag_len
)
1486 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
1488 netif_err(efx
, hw
, efx
->net_dev
,
1489 "PTP unexpected event length: got %d expected %d\n",
1490 ptp
->evt_frag_idx
, expected_frag_len
);
1491 ptp
->reset_required
= true;
1492 queue_work(ptp
->workwq
, &ptp
->work
);
1495 /* Process a completed receive event. Put it on the event queue and
1496 * start worker thread. This is required because event and their
1497 * correspoding packets may come in either order.
1499 static void ptp_event_rx(struct efx_nic
*efx
, struct efx_ptp_data
*ptp
)
1501 struct efx_ptp_event_rx
*evt
= NULL
;
1503 if (WARN_ON_ONCE(ptp
->rx_ts_inline
))
1506 if (ptp
->evt_frag_idx
!= 3) {
1507 ptp_event_failure(efx
, 3);
1511 spin_lock_bh(&ptp
->evt_lock
);
1512 if (!list_empty(&ptp
->evt_free_list
)) {
1513 evt
= list_first_entry(&ptp
->evt_free_list
,
1514 struct efx_ptp_event_rx
, link
);
1515 list_del(&evt
->link
);
1517 evt
->seq0
= EFX_QWORD_FIELD(ptp
->evt_frags
[2], MCDI_EVENT_DATA
);
1518 evt
->seq1
= (EFX_QWORD_FIELD(ptp
->evt_frags
[2],
1520 (EFX_QWORD_FIELD(ptp
->evt_frags
[1],
1521 MCDI_EVENT_SRC
) << 8) |
1522 (EFX_QWORD_FIELD(ptp
->evt_frags
[0],
1523 MCDI_EVENT_SRC
) << 16));
1524 evt
->hwtimestamp
= efx
->ptp_data
->nic_to_kernel_time(
1525 EFX_QWORD_FIELD(ptp
->evt_frags
[0], MCDI_EVENT_DATA
),
1526 EFX_QWORD_FIELD(ptp
->evt_frags
[1], MCDI_EVENT_DATA
),
1527 ptp
->ts_corrections
.rx
);
1528 evt
->expiry
= jiffies
+ msecs_to_jiffies(PKT_EVENT_LIFETIME_MS
);
1529 list_add_tail(&evt
->link
, &ptp
->evt_list
);
1531 queue_work(ptp
->workwq
, &ptp
->work
);
1532 } else if (!ptp
->evt_overflow
) {
1533 /* Log a warning message and set the event overflow flag.
1534 * The message won't be logged again until the event queue
1537 netif_err(efx
, rx_err
, efx
->net_dev
, "PTP event queue overflow\n");
1538 ptp
->evt_overflow
= true;
1540 spin_unlock_bh(&ptp
->evt_lock
);
1543 static void ptp_event_fault(struct efx_nic
*efx
, struct efx_ptp_data
*ptp
)
1545 int code
= EFX_QWORD_FIELD(ptp
->evt_frags
[0], MCDI_EVENT_DATA
);
1546 if (ptp
->evt_frag_idx
!= 1) {
1547 ptp_event_failure(efx
, 1);
1551 netif_err(efx
, hw
, efx
->net_dev
, "PTP error %d\n", code
);
1554 static void ptp_event_pps(struct efx_nic
*efx
, struct efx_ptp_data
*ptp
)
1556 if (ptp
->nic_ts_enabled
)
1557 queue_work(ptp
->pps_workwq
, &ptp
->pps_work
);
1560 void efx_ptp_event(struct efx_nic
*efx
, efx_qword_t
*ev
)
1562 struct efx_ptp_data
*ptp
= efx
->ptp_data
;
1563 int code
= EFX_QWORD_FIELD(*ev
, MCDI_EVENT_CODE
);
1568 if (ptp
->evt_frag_idx
== 0) {
1569 ptp
->evt_code
= code
;
1570 } else if (ptp
->evt_code
!= code
) {
1571 netif_err(efx
, hw
, efx
->net_dev
,
1572 "PTP out of sequence event %d\n", code
);
1573 ptp
->evt_frag_idx
= 0;
1576 ptp
->evt_frags
[ptp
->evt_frag_idx
++] = *ev
;
1577 if (!MCDI_EVENT_FIELD(*ev
, CONT
)) {
1578 /* Process resulting event */
1580 case MCDI_EVENT_CODE_PTP_RX
:
1581 ptp_event_rx(efx
, ptp
);
1583 case MCDI_EVENT_CODE_PTP_FAULT
:
1584 ptp_event_fault(efx
, ptp
);
1586 case MCDI_EVENT_CODE_PTP_PPS
:
1587 ptp_event_pps(efx
, ptp
);
1590 netif_err(efx
, hw
, efx
->net_dev
,
1591 "PTP unknown event %d\n", code
);
1594 ptp
->evt_frag_idx
= 0;
1595 } else if (MAX_EVENT_FRAGS
== ptp
->evt_frag_idx
) {
1596 netif_err(efx
, hw
, efx
->net_dev
,
1597 "PTP too many event fragments\n");
1598 ptp
->evt_frag_idx
= 0;
1602 void efx_time_sync_event(struct efx_channel
*channel
, efx_qword_t
*ev
)
1604 channel
->sync_timestamp_major
= MCDI_EVENT_FIELD(*ev
, PTP_TIME_MAJOR
);
1605 channel
->sync_timestamp_minor
=
1606 MCDI_EVENT_FIELD(*ev
, PTP_TIME_MINOR_26_19
) << 19;
1607 /* if sync events have been disabled then we want to silently ignore
1608 * this event, so throw away result.
1610 (void) cmpxchg(&channel
->sync_events_state
, SYNC_EVENTS_REQUESTED
,
1614 /* make some assumptions about the time representation rather than abstract it,
1615 * since we currently only support one type of inline timestamping and only on
1618 #define MINOR_TICKS_PER_SECOND 0x8000000
1619 /* Fuzz factor for sync events to be out of order with RX events */
1620 #define FUZZ (MINOR_TICKS_PER_SECOND / 10)
1621 #define EXPECTED_SYNC_EVENTS_PER_SECOND 4
1623 static inline u32
efx_rx_buf_timestamp_minor(struct efx_nic
*efx
, const u8
*eh
)
1625 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
1626 return __le32_to_cpup((const __le32
*)(eh
+ efx
->rx_packet_ts_offset
));
1628 const u8
*data
= eh
+ efx
->rx_packet_ts_offset
;
1629 return (u32
)data
[0] |
1631 (u32
)data
[2] << 16 |
1636 void __efx_rx_skb_attach_timestamp(struct efx_channel
*channel
,
1637 struct sk_buff
*skb
)
1639 struct efx_nic
*efx
= channel
->efx
;
1640 u32 pkt_timestamp_major
, pkt_timestamp_minor
;
1642 struct skb_shared_hwtstamps
*timestamps
;
1644 pkt_timestamp_minor
= (efx_rx_buf_timestamp_minor(efx
,
1645 skb_mac_header(skb
)) +
1646 (u32
) efx
->ptp_data
->ts_corrections
.rx
) &
1647 (MINOR_TICKS_PER_SECOND
- 1);
1649 /* get the difference between the packet and sync timestamps,
1652 diff
= (pkt_timestamp_minor
- channel
->sync_timestamp_minor
) &
1653 (MINOR_TICKS_PER_SECOND
- 1);
1654 /* do we roll over a second boundary and need to carry the one? */
1655 carry
= channel
->sync_timestamp_minor
+ diff
> MINOR_TICKS_PER_SECOND
?
1658 if (diff
<= MINOR_TICKS_PER_SECOND
/ EXPECTED_SYNC_EVENTS_PER_SECOND
+
1660 /* packet is ahead of the sync event by a quarter of a second or
1661 * less (allowing for fuzz)
1663 pkt_timestamp_major
= channel
->sync_timestamp_major
+ carry
;
1664 } else if (diff
>= MINOR_TICKS_PER_SECOND
- FUZZ
) {
1665 /* packet is behind the sync event but within the fuzz factor.
1666 * This means the RX packet and sync event crossed as they were
1667 * placed on the event queue, which can sometimes happen.
1669 pkt_timestamp_major
= channel
->sync_timestamp_major
- 1 + carry
;
1671 /* it's outside tolerance in both directions. this might be
1672 * indicative of us missing sync events for some reason, so
1673 * we'll call it an error rather than risk giving a bogus
1676 netif_vdbg(efx
, drv
, efx
->net_dev
,
1677 "packet timestamp %x too far from sync event %x:%x\n",
1678 pkt_timestamp_minor
, channel
->sync_timestamp_major
,
1679 channel
->sync_timestamp_minor
);
1683 /* attach the timestamps to the skb */
1684 timestamps
= skb_hwtstamps(skb
);
1685 timestamps
->hwtstamp
=
1686 efx_ptp_s27_to_ktime(pkt_timestamp_major
, pkt_timestamp_minor
);
1689 static int efx_phc_adjfreq(struct ptp_clock_info
*ptp
, s32 delta
)
1691 struct efx_ptp_data
*ptp_data
= container_of(ptp
,
1692 struct efx_ptp_data
,
1694 struct efx_nic
*efx
= ptp_data
->efx
;
1695 MCDI_DECLARE_BUF(inadj
, MC_CMD_PTP_IN_ADJUST_LEN
);
1699 if (delta
> MAX_PPB
)
1701 else if (delta
< -MAX_PPB
)
1704 /* Convert ppb to fixed point ns. */
1705 adjustment_ns
= (((s64
)delta
* PPB_SCALE_WORD
) >>
1706 (PPB_EXTRA_BITS
+ MAX_PPB_BITS
));
1708 MCDI_SET_DWORD(inadj
, PTP_IN_OP
, MC_CMD_PTP_OP_ADJUST
);
1709 MCDI_SET_DWORD(inadj
, PTP_IN_PERIPH_ID
, 0);
1710 MCDI_SET_QWORD(inadj
, PTP_IN_ADJUST_FREQ
, adjustment_ns
);
1711 MCDI_SET_DWORD(inadj
, PTP_IN_ADJUST_SECONDS
, 0);
1712 MCDI_SET_DWORD(inadj
, PTP_IN_ADJUST_NANOSECONDS
, 0);
1713 rc
= efx_mcdi_rpc(efx
, MC_CMD_PTP
, inadj
, sizeof(inadj
),
1718 ptp_data
->current_adjfreq
= adjustment_ns
;
1722 static int efx_phc_adjtime(struct ptp_clock_info
*ptp
, s64 delta
)
1724 u32 nic_major
, nic_minor
;
1725 struct efx_ptp_data
*ptp_data
= container_of(ptp
,
1726 struct efx_ptp_data
,
1728 struct efx_nic
*efx
= ptp_data
->efx
;
1729 MCDI_DECLARE_BUF(inbuf
, MC_CMD_PTP_IN_ADJUST_LEN
);
1731 efx
->ptp_data
->ns_to_nic_time(delta
, &nic_major
, &nic_minor
);
1733 MCDI_SET_DWORD(inbuf
, PTP_IN_OP
, MC_CMD_PTP_OP_ADJUST
);
1734 MCDI_SET_DWORD(inbuf
, PTP_IN_PERIPH_ID
, 0);
1735 MCDI_SET_QWORD(inbuf
, PTP_IN_ADJUST_FREQ
, ptp_data
->current_adjfreq
);
1736 MCDI_SET_DWORD(inbuf
, PTP_IN_ADJUST_MAJOR
, nic_major
);
1737 MCDI_SET_DWORD(inbuf
, PTP_IN_ADJUST_MINOR
, nic_minor
);
1738 return efx_mcdi_rpc(efx
, MC_CMD_PTP
, inbuf
, sizeof(inbuf
),
1742 static int efx_phc_gettime(struct ptp_clock_info
*ptp
, struct timespec
*ts
)
1744 struct efx_ptp_data
*ptp_data
= container_of(ptp
,
1745 struct efx_ptp_data
,
1747 struct efx_nic
*efx
= ptp_data
->efx
;
1748 MCDI_DECLARE_BUF(inbuf
, MC_CMD_PTP_IN_READ_NIC_TIME_LEN
);
1749 MCDI_DECLARE_BUF(outbuf
, MC_CMD_PTP_OUT_READ_NIC_TIME_LEN
);
1753 MCDI_SET_DWORD(inbuf
, PTP_IN_OP
, MC_CMD_PTP_OP_READ_NIC_TIME
);
1754 MCDI_SET_DWORD(inbuf
, PTP_IN_PERIPH_ID
, 0);
1756 rc
= efx_mcdi_rpc(efx
, MC_CMD_PTP
, inbuf
, sizeof(inbuf
),
1757 outbuf
, sizeof(outbuf
), NULL
);
1761 kt
= ptp_data
->nic_to_kernel_time(
1762 MCDI_DWORD(outbuf
, PTP_OUT_READ_NIC_TIME_MAJOR
),
1763 MCDI_DWORD(outbuf
, PTP_OUT_READ_NIC_TIME_MINOR
), 0);
1764 *ts
= ktime_to_timespec(kt
);
1768 static int efx_phc_settime(struct ptp_clock_info
*ptp
,
1769 const struct timespec
*e_ts
)
1771 /* Get the current NIC time, efx_phc_gettime.
1772 * Subtract from the desired time to get the offset
1773 * call efx_phc_adjtime with the offset
1776 struct timespec time_now
;
1777 struct timespec delta
;
1779 rc
= efx_phc_gettime(ptp
, &time_now
);
1783 delta
= timespec_sub(*e_ts
, time_now
);
1785 rc
= efx_phc_adjtime(ptp
, timespec_to_ns(&delta
));
1792 static int efx_phc_enable(struct ptp_clock_info
*ptp
,
1793 struct ptp_clock_request
*request
,
1796 struct efx_ptp_data
*ptp_data
= container_of(ptp
,
1797 struct efx_ptp_data
,
1799 if (request
->type
!= PTP_CLK_REQ_PPS
)
1802 ptp_data
->nic_ts_enabled
= !!enable
;
1806 static const struct efx_channel_type efx_ptp_channel_type
= {
1807 .handle_no_channel
= efx_ptp_handle_no_channel
,
1808 .pre_probe
= efx_ptp_probe_channel
,
1809 .post_remove
= efx_ptp_remove_channel
,
1810 .get_name
= efx_ptp_get_channel_name
,
1811 /* no copy operation; there is no need to reallocate this channel */
1812 .receive_skb
= efx_ptp_rx
,
1813 .keep_eventq
= false,
1816 void efx_ptp_defer_probe_with_channel(struct efx_nic
*efx
)
1818 /* Check whether PTP is implemented on this NIC. The DISABLE
1819 * operation will succeed if and only if it is implemented.
1821 if (efx_ptp_disable(efx
) == 0)
1822 efx
->extra_channel_type
[EFX_EXTRA_CHANNEL_PTP
] =
1823 &efx_ptp_channel_type
;
1826 void efx_ptp_start_datapath(struct efx_nic
*efx
)
1828 if (efx_ptp_restart(efx
))
1829 netif_err(efx
, drv
, efx
->net_dev
, "Failed to restart PTP.\n");
1830 /* re-enable timestamping if it was previously enabled */
1831 if (efx
->type
->ptp_set_ts_sync_events
)
1832 efx
->type
->ptp_set_ts_sync_events(efx
, true, true);
1835 void efx_ptp_stop_datapath(struct efx_nic
*efx
)
1837 /* temporarily disable timestamping */
1838 if (efx
->type
->ptp_set_ts_sync_events
)
1839 efx
->type
->ptp_set_ts_sync_events(efx
, false, true);