diff --git a/include/quicly.h b/include/quicly.h index 519682c8..0d5fcaba 100644 --- a/include/quicly.h +++ b/include/quicly.h @@ -946,8 +946,10 @@ typedef struct st_quicly_stream_callbacks_t { * asks the application to fill the frame payload. `off` is the offset within the buffer (the beginning position of the buffer * changes as `on_send_shift` is invoked). `len` is an in/out argument that specifies the size of the buffer / amount of data * being written. `wrote_all` is a boolean out parameter indicating if the application has written all the available data. - * As this callback is triggered by calling quicly_stream_sync_sendbuf (stream, 1) when tx data is present, it assumes data - * to be available - that is `len` return value should be non-zero. + * This callback typically writes some data and therefore sets `*len` to a non-zero value, as the callback is triggered by + * calling quicly_stream_sync_sendbuf (stream, 1) when tx data is present. However, the callback may set `*len` to zero to + * indicate that payload was not immediately avaiable (e,g., when it has to be loaded from disk). It is the responsibility of + * the user-supplied stream scheduler to reschedule the stream once data is loaded. */ void (*on_send_emit)(quicly_stream_t *stream, size_t off, void *dst, size_t *len, int *wrote_all); /** @@ -996,6 +998,12 @@ struct st_quicly_stream_t { * */ unsigned streams_blocked : 1; + /** + * if `on_send_emit` should be called to read payload of multiple datagrams. After the callback returns, quicly calls `memmove` + * to scatter the payload. Setting this flag is likely to improve throughput when the overhead is high for reading the payload + * (e.g., when `on_send_emit` calls `pread`). + */ + unsigned scatter_emit : 1; /** * */ diff --git a/include/quicly/constants.h b/include/quicly/constants.h index 3b210786..be5aa1d7 100644 --- a/include/quicly/constants.h +++ b/include/quicly/constants.h @@ -118,6 +118,7 @@ typedef int64_t quicly_error_t; #define QUICLY_ERROR_STATE_EXHAUSTION 0xff07 #define QUICLY_ERROR_INVALID_INITIAL_VERSION 0xff08 #define QUICLY_ERROR_DECRYPTION_FAILED 0xff09 +#define QUICLY_ERROR_SEND_EMIT_BLOCKED 0xff0a typedef int64_t quicly_stream_id_t; diff --git a/lib/quicly.c b/lib/quicly.c index ffdc028f..951e7d9d 100644 --- a/lib/quicly.c +++ b/lib/quicly.c @@ -1253,6 +1253,8 @@ static void init_stream_properties(quicly_stream_t *stream, uint32_t initial_max } stream->streams_blocked = 0; + stream->scatter_emit = 0; + stream->_send_aux.max_stream_data = initial_max_stream_data_remote; stream->_send_aux.stop_sending.sender_state = QUICLY_SENDER_STATE_NONE; stream->_send_aux.stop_sending.error_code = 0; @@ -3986,14 +3988,14 @@ static inline uint8_t *emit_cid(uint8_t *dst, const quicly_cid_t *cid) return dst; } -enum allocate_frame_type { - ALLOCATE_FRAME_TYPE_NON_ACK_ELICITING, - ALLOCATE_FRAME_TYPE_ACK_ELICITING, - ALLOCATE_FRAME_TYPE_ACK_ELICITING_NO_CC, -}; +#define ALLOCATE_FRAME_FLAG_CONSULT_CC 0x1 +#define ALLOCATE_FRAME_FLAG_ADJUST_ACK_FREQUENCY 0x2 -static quicly_error_t do_allocate_frame(quicly_conn_t *conn, quicly_send_context_t *s, size_t min_space, - enum allocate_frame_type frame_type) +/** + * Allocates a frame. This is a low-level function; for allocating ordinary ACK-eliciting frames, use `allocate_ack_eliciting_frame` + * instead. + */ +static quicly_error_t allocate_frame(quicly_conn_t *conn, quicly_send_context_t *s, size_t min_space, unsigned flags) { int coalescible; quicly_error_t ret; @@ -4043,7 +4045,7 @@ static quicly_error_t do_allocate_frame(quicly_conn_t *conn, quicly_send_context if (s->num_datagrams >= s->max_datagrams) return QUICLY_ERROR_SENDBUF_FULL; /* note: send_window (ssize_t) can become negative; see doc-comment */ - if (frame_type == ALLOCATE_FRAME_TYPE_ACK_ELICITING && s->send_window <= 0) + if ((flags & ALLOCATE_FRAME_FLAG_CONSULT_CC) != 0 && s->send_window <= 0) return QUICLY_ERROR_SENDBUF_FULL; if (s->payload_buf.end - s->payload_buf.datagram < conn->egress.max_udp_payload_size) return QUICLY_ERROR_SENDBUF_FULL; @@ -4098,7 +4100,7 @@ static quicly_error_t do_allocate_frame(quicly_conn_t *conn, quicly_send_context if ((ret = quicly_sentmap_prepare(&conn->egress.loss.sentmap, conn->egress.packet_number, conn->stash.now, ack_epoch)) != 0) return ret; /* adjust ack-frequency */ - if (frame_type == ALLOCATE_FRAME_TYPE_ACK_ELICITING && conn->stash.now >= conn->egress.ack_frequency.update_at && + if ((flags & ALLOCATE_FRAME_FLAG_ADJUST_ACK_FREQUENCY) != 0 && conn->stash.now >= conn->egress.ack_frequency.update_at && s->dst_end - s->dst >= QUICLY_ACK_FREQUENCY_FRAME_CAPACITY + min_space) { assert(conn->super.remote.transport_params.min_ack_delay_usec != UINT64_MAX); if (conn->egress.cc.num_loss_episodes >= QUICLY_FIRST_ACK_FREQUENCY_LOSS_EPISODE && conn->initial == NULL && @@ -4124,21 +4126,24 @@ static quicly_error_t do_allocate_frame(quicly_conn_t *conn, quicly_send_context } TargetReady: - if (frame_type != ALLOCATE_FRAME_TYPE_NON_ACK_ELICITING) { - s->target.ack_eliciting = 1; - conn->egress.last_retransmittable_sent_at = conn->stash.now; - } return 0; } -static quicly_error_t allocate_ack_eliciting_frame(quicly_conn_t *conn, quicly_send_context_t *s, size_t min_space, - quicly_sent_t **sent, quicly_sent_acked_cb acked) +static void mark_frame_built_as_ack_eliciting(quicly_conn_t *conn, quicly_send_context_t *s) +{ + s->target.ack_eliciting = 1; + conn->egress.last_retransmittable_sent_at = conn->stash.now; +} + +static inline quicly_error_t allocate_ack_eliciting_frame(quicly_conn_t *conn, quicly_send_context_t *s, size_t min_space, + quicly_sent_t **sent, quicly_sent_acked_cb acked) { quicly_error_t ret; - if ((ret = do_allocate_frame(conn, s, min_space, ALLOCATE_FRAME_TYPE_ACK_ELICITING)) != 0) + if ((ret = allocate_frame(conn, s, min_space, ALLOCATE_FRAME_FLAG_CONSULT_CC | ALLOCATE_FRAME_FLAG_ADJUST_ACK_FREQUENCY)) != 0) return ret; - if ((*sent = quicly_sentmap_allocate(&conn->egress.loss.sentmap, acked)) == NULL) + mark_frame_built_as_ack_eliciting(conn, s); + if (sent != NULL && (*sent = quicly_sentmap_allocate(&conn->egress.loss.sentmap, acked)) == NULL) return PTLS_ERROR_NO_MEMORY; return ret; @@ -4162,7 +4167,7 @@ static quicly_error_t send_ack(quicly_conn_t *conn, struct st_quicly_pn_space_t } Emit: /* emit an ACK frame */ - if ((ret = do_allocate_frame(conn, s, QUICLY_ACK_FRAME_CAPACITY, ALLOCATE_FRAME_TYPE_NON_ACK_ELICITING)) != 0) + if ((ret = allocate_frame(conn, s, QUICLY_ACK_FRAME_CAPACITY, 0)) != 0) return ret; uint8_t *dst = s->dst; dst = quicly_encode_ack_frame(dst, s->dst_end, &space->ack_queue, space->ecn_counts, ack_delay); @@ -4398,48 +4403,178 @@ int quicly_can_send_data(quicly_conn_t *conn, quicly_send_context_t *s) return s->num_datagrams < s->max_datagrams; } +static void commit_stream_frame(quicly_stream_t *stream, quicly_sent_t *sent, uint64_t off, const uint8_t *data, size_t len, + int wrote_all, int is_fin) +{ + /* update and log per-frame stats */ + if (stream->stream_id < 0) { + ++stream->conn->super.stats.num_frames_sent.crypto; + } else { + ++stream->conn->super.stats.num_frames_sent.stream; + } + QUICLY_PROBE(STREAM_SEND, stream->conn, stream->conn->stash.now, stream, off, data, len, is_fin, wrote_all); + QUICLY_LOG_CONN(stream_send, stream->conn, { + PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id); + PTLS_LOG_ELEMENT_UNSIGNED(off, off); + PTLS_LOG_APPDATA_ELEMENT_HEXDUMP(data, data, len); + PTLS_LOG_ELEMENT_BOOL(is_fin, is_fin); + PTLS_LOG_ELEMENT_BOOL(wrote_all, wrote_all); + }); + QUICLY_PROBE(QUICTRACE_SEND_STREAM, stream->conn, stream->conn->stash.now, stream, off, len, is_fin); + + /* setup sentmap */ + sent->data.stream.stream_id = stream->stream_id; + sent->data.stream.args.start = off; + sent->data.stream.args.end = off + len + is_fin; +} + +static quicly_error_t update_stream_sendstate(quicly_stream_t *stream, uint64_t off, size_t len, int is_fin, int wrote_all) +{ + quicly_error_t ret; + + stream->conn->super.stats.num_bytes.stream_data_sent += len; + if (off < stream->sendstate.size_inflight) + stream->conn->super.stats.num_bytes.stream_data_resent += + (stream->sendstate.size_inflight < off + len ? stream->sendstate.size_inflight : off + len) - off; + + if (stream->sendstate.size_inflight < off + len) { + if (stream->stream_id >= 0) + stream->conn->egress.max_data.sent += off + len - stream->sendstate.size_inflight; + stream->sendstate.size_inflight = off + len; + } + if ((ret = quicly_ranges_subtract(&stream->sendstate.pending, off, off + len + is_fin)) != 0) + return ret; + if (wrote_all) { + if ((ret = quicly_ranges_subtract(&stream->sendstate.pending, stream->sendstate.size_inflight, UINT64_MAX)) != 0) + return ret; + } + + return ret; +} + /** - * If necessary, changes the frame representation from one without length field to one that has if necessary. Or, as an alternative, - * prepends PADDING frames. Upon return, `dst` points to the end of the frame being built. `*len`, `*wrote_all`, `*frame_type_at` - * are also updated reflecting their values post-adjustment. + * Assuming `dst` points to where the Length field of a CRYPTO frame should be inserted, inserts the field, changing `*len` and + * `*wrote_all` if necessary. Returns the end of the CRYPTO frame being adjusted. */ -static inline void adjust_stream_frame_layout(uint8_t **dst, uint8_t *const dst_end, size_t *len, int *wrote_all, - uint8_t **frame_at) +static uint8_t *adjust_crypto_frame_layout(uint8_t *dst, uint8_t *const dst_end, size_t *len, int *wrote_all) { - size_t space_left = (dst_end - *dst) - *len, len_of_len = quicly_encodev_capacity(*len); + size_t space_left = (dst_end - dst) - *len, len_of_len = quicly_encodev_capacity(*len); - if (**frame_at == QUICLY_FRAME_TYPE_CRYPTO) { - /* CRYPTO frame: adjust payload length to make space for the length field, if necessary. */ - if (space_left < len_of_len) { - *len = dst_end - *dst - len_of_len; - *wrote_all = 0; - } - } else { - /* STREAM frame: insert length if space can be left for more frames. Otherwise, retain STREAM frame header omitting the - * length field, prepending PADDING if necessary. */ - if (space_left <= len_of_len) { - if (space_left != 0) { - memmove(*frame_at + space_left, *frame_at, *dst + *len - *frame_at); - memset(*frame_at, QUICLY_FRAME_TYPE_PADDING, space_left); - *dst += space_left; - *frame_at += space_left; - } - *dst += *len; - return; - } - **frame_at |= QUICLY_FRAME_TYPE_STREAM_BIT_LEN; + if (space_left < len_of_len) { + *len = dst_end - dst - len_of_len; + *wrote_all = 0; } /* insert length before payload of `*len` bytes */ - memmove(*dst + len_of_len, *dst, *len); - *dst = quicly_encodev(*dst, *len); - *dst += *len; + memmove(dst + len_of_len, dst, *len); + dst = quicly_encodev(dst, *len); + dst += *len; + + return dst; +} + +/** + * Assuming that `header` and `header_len` point to a STREAM frame without a Length field, either inserts a Length field or prepends + * a PADDING frame if necessary. Returns the size increase of the header. + */ +static size_t adjust_last_stream_frame(uint8_t *header, size_t header_len, uint16_t payload_size, size_t space_left, + int move_payload) +{ + if (space_left == 0) + return 0; + + size_t len_len = quicly_encodev_capacity(payload_size); + + if (space_left <= len_len) { + /* prepend PADDING, as there is not enough space to insert the length field */ + if (move_payload) + memmove(header + header_len + space_left, header + header_len, payload_size); + memmove(header + space_left, header, header_len); + memset(header, QUICLY_FRAME_TYPE_PADDING, space_left); + return space_left; + } + + /* add the Length field */ + if (move_payload) + memmove(header + header_len + len_len, header + header_len, payload_size); + header[0] |= QUICLY_FRAME_TYPE_STREAM_BIT_LEN; + quicly_encodev(header + header_len, payload_size); + return len_len; +} + +#define STREAM_SEND_MAX_EXTRA_DATAGRAMS 9 /* max burst size (of 10 packets) - 1 */ + +/** + * Adjusts the STREAM frame layout. If given payload expands beyond the end of the current datagram, scatters the payload to the + * correct locations assuming that more datagrams would be built adjacently. STREAM headers are prepended to the scattered payload. + * Returns the end of the last stream frame. + */ +static uint8_t *scatter_stream_payload(quicly_send_context_t *s, uint16_t datagram_size, quicly_stream_id_t stream_id, + uint64_t stream_start, uint8_t *payload_start, size_t *len, int *wrote_all, + uint16_t *scattered_payload_lengths) +{ + /* fast path when no expansion is required; adjust the layout and return */ + if (*len <= s->dst_end - payload_start) { + size_t space_left = s->dst_end - (payload_start + *len); + size_t add_space = adjust_last_stream_frame(s->dst, payload_start - s->dst, *len, space_left, 1); + payload_start += add_space; + scattered_payload_lengths[0] = 0; + return payload_start + *len; + } + + struct { + uint8_t len; + uint8_t bytes[1 + 8 + 8 + 2]; + } frame_headers[STREAM_SEND_MAX_EXTRA_DATAGRAMS]; + uint64_t stream_offset = stream_start + (s->dst_end - payload_start), stream_end = stream_start + *len; + size_t num_scattered, datagram_prefix_len = 1 /* header byte */ + s->dcid->len + QUICLY_SEND_PN_SIZE, + datagram_capacity = datagram_size - datagram_prefix_len - s->current.cipher->aead->algo->tag_size; + + /* build frame headers for the extra datagrams, calculating their offsets */ + for (num_scattered = 0; stream_offset < stream_end && num_scattered < PTLS_ELEMENTSOF(frame_headers); ++num_scattered) { + uint8_t *hp = frame_headers[num_scattered].bytes; + *hp++ = QUICLY_FRAME_TYPE_STREAM_BASE | QUICLY_FRAME_TYPE_STREAM_BIT_OFF; + hp = quicly_encodev(hp, stream_id); + hp = quicly_encodev(hp, stream_offset); + frame_headers[num_scattered].len = hp - frame_headers[num_scattered].bytes; + scattered_payload_lengths[num_scattered] = datagram_capacity - frame_headers[num_scattered].len; + if (scattered_payload_lengths[num_scattered] > stream_end - stream_offset) + scattered_payload_lengths[num_scattered] = stream_end - stream_offset; + stream_offset += scattered_payload_lengths[num_scattered]; + } + scattered_payload_lengths[num_scattered] = 0; + + { /* adjust the encoding of the last frame */ + size_t space_left = + datagram_capacity - (frame_headers[num_scattered - 1].len + scattered_payload_lengths[num_scattered - 1]); + frame_headers[num_scattered - 1].len += + adjust_last_stream_frame(frame_headers[num_scattered - 1].bytes, frame_headers[num_scattered - 1].len, + scattered_payload_lengths[num_scattered - 1], space_left, 0); + } + + /* adjust out parameters, move stream payload, and write frame headers for the expanded datagrams */ + if (stream_offset - stream_start < *len) { + *wrote_all = 0; + *len = stream_offset - stream_start; + } + + uint8_t *extra_packets_from = s->dst_end + s->current.cipher->aead->algo->tag_size; + + /* move stream payload and write headers */ + for (size_t i = num_scattered - 1; i != SIZE_MAX; --i) { + stream_offset -= scattered_payload_lengths[i]; + memmove(extra_packets_from + datagram_size * i + datagram_prefix_len + frame_headers[i].len, + payload_start + stream_offset - stream_start, scattered_payload_lengths[i]); + memcpy(extra_packets_from + datagram_size * i + datagram_prefix_len, frame_headers[i].bytes, frame_headers[i].len); + } + + return extra_packets_from + datagram_size * (num_scattered - 1) + datagram_prefix_len + frame_headers[num_scattered - 1].len + + scattered_payload_lengths[num_scattered - 1]; } quicly_error_t quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s) { uint64_t off = stream->sendstate.pending.ranges[0].start; - quicly_sent_t *sent; uint8_t *dst; /* this pointer points to the current write position within the frame being built, while `s->dst` points to the * beginning of the frame. */ size_t len; @@ -4448,9 +4583,8 @@ quicly_error_t quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t /* write frame type, stream_id and offset, calculate capacity (and store that in `len`) */ if (stream->stream_id < 0) { - if ((ret = allocate_ack_eliciting_frame(stream->conn, s, - 1 + quicly_encodev_capacity(off) + 2 /* type + offset + len + 1-byte payload */, - &sent, on_ack_stream)) != 0) + if ((ret = allocate_frame(stream->conn, s, 1 + quicly_encodev_capacity(off) + 2 /* type + offset + len + 1-byte payload */, + ALLOCATE_FRAME_FLAG_CONSULT_CC | ALLOCATE_FRAME_FLAG_ADJUST_ACK_FREQUENCY)) != 0) return ret; dst = s->dst; *dst++ = QUICLY_FRAME_TYPE_CRYPTO; @@ -4466,8 +4600,9 @@ quicly_error_t quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t header[0] = QUICLY_FRAME_TYPE_STREAM_BASE; } if (off == stream->sendstate.final_size) { - assert(!quicly_sendstate_is_open(&stream->sendstate)); /* special case for emitting FIN only */ + quicly_sent_t *sent; + assert(!quicly_sendstate_is_open(&stream->sendstate)); header[0] |= QUICLY_FRAME_TYPE_STREAM_BIT_FIN; if ((ret = allocate_ack_eliciting_frame(stream->conn, s, hp - header, &sent, on_ack_stream)) != 0) return ret; @@ -4477,17 +4612,30 @@ quicly_error_t quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t } memcpy(s->dst, header, hp - header); s->dst += hp - header; - len = 0; - wrote_all = 1; - is_fin = 1; - goto UpdateState; + commit_stream_frame(stream, sent, off, s->dst, 0, 1, 1); + update_stream_sendstate(stream, off, 0, 1, 1); + return 0; } - if ((ret = allocate_ack_eliciting_frame(stream->conn, s, hp - header + 1, &sent, on_ack_stream)) != 0) + if ((ret = allocate_frame(stream->conn, s, hp - header + 1, + ALLOCATE_FRAME_FLAG_CONSULT_CC | ALLOCATE_FRAME_FLAG_ADJUST_ACK_FREQUENCY)) != 0) return ret; dst = s->dst; memcpy(dst, header, hp - header); dst += hp - header; len = s->dst_end - dst; + /* if sending in 1-RTT, generate stream payload past the end of current datagram and move them to build datagrams */ + if (get_epoch(s->current.first_byte) == QUICLY_EPOCH_1RTT) { + size_t mtu = stream->conn->egress.max_udp_payload_size; + size_t extra_datagrams = s->send_window > mtu ? (s->send_window + mtu - 1) / mtu - 1 : 0; + if (extra_datagrams > (s->payload_buf.end - (s->dst_end + s->target.cipher->aead->algo->tag_size)) / mtu) + extra_datagrams = (s->payload_buf.end - (s->dst_end + s->target.cipher->aead->algo->tag_size)) / mtu; + if (extra_datagrams > s->max_datagrams - s->num_datagrams - 1) + extra_datagrams = s->max_datagrams - s->num_datagrams - 1; + if (extra_datagrams > STREAM_SEND_MAX_EXTRA_DATAGRAMS) + extra_datagrams = STREAM_SEND_MAX_EXTRA_DATAGRAMS; + size_t overhead = 1 + s->dcid->len + QUICLY_SEND_PN_SIZE + (dst - s->dst) + s->current.cipher->aead->algo->tag_size; + len += (mtu - overhead) * extra_datagrams; + } /* cap by max_stream_data */ if (off + len > stream->_send_aux.max_stream_data) len = stream->_send_aux.max_stream_data - off; @@ -4528,63 +4676,80 @@ quicly_error_t quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t return QUICLY_ERROR_IS_CLOSING; } else if (stream->_send_aux.reset_stream.sender_state != QUICLY_SENDER_STATE_NONE) { return 0; - } - assert(len != 0); - - adjust_stream_frame_layout(&dst, s->dst_end, &len, &wrote_all, &s->dst); - - /* determine if the frame incorporates FIN */ - if (off + len == stream->sendstate.final_size) { - assert(!quicly_sendstate_is_open(&stream->sendstate)); - assert(s->dst != NULL); - is_fin = 1; - *s->dst |= QUICLY_FRAME_TYPE_STREAM_BIT_FIN; - } else { - is_fin = 0; + } else if (len == 0) { + assert(!wrote_all); /* Do we want to allow on_send_emit to indicate steram closure without writing anything? */ + return QUICLY_ERROR_SEND_EMIT_BLOCKED; } - /* update s->dst now that frame construction is complete */ - s->dst = dst; + /* Finally, we are certain that a frame is built. Mark the packet as ack-elicting and allocate a sentmap entry. */ + mark_frame_built_as_ack_eliciting(stream->conn, s); + quicly_sent_t *sent; + if ((sent = quicly_sentmap_allocate(&stream->conn->egress.loss.sentmap, on_ack_stream)) == NULL) + return PTLS_ERROR_NO_MEMORY; -UpdateState: + /* Adjust the frame layout and commit. */ if (stream->stream_id < 0) { - ++stream->conn->super.stats.num_frames_sent.crypto; + /* CRYPTO frame */ + s->dst = adjust_crypto_frame_layout(dst, s->dst_end, &len, &wrote_all); + commit_stream_frame(stream, sent, off, s->dst - len, len, wrote_all, 0); + is_fin = 0; } else { - ++stream->conn->super.stats.num_frames_sent.stream; - } - stream->conn->super.stats.num_bytes.stream_data_sent += len; - if (off < stream->sendstate.size_inflight) - stream->conn->super.stats.num_bytes.stream_data_resent += - (stream->sendstate.size_inflight < off + len ? stream->sendstate.size_inflight : off + len) - off; - QUICLY_PROBE(STREAM_SEND, stream->conn, stream->conn->stash.now, stream, off, s->dst - len, len, is_fin, wrote_all); - QUICLY_LOG_CONN(stream_send, stream->conn, { - PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id); - PTLS_LOG_ELEMENT_UNSIGNED(off, off); - PTLS_LOG_APPDATA_ELEMENT_HEXDUMP(data, s->dst - len, len); - PTLS_LOG_ELEMENT_BOOL(is_fin, is_fin); - PTLS_LOG_ELEMENT_BOOL(wrote_all, wrote_all); - }); - - QUICLY_PROBE(QUICTRACE_SEND_STREAM, stream->conn, stream->conn->stash.now, stream, off, len, is_fin); - /* update sendstate (and also MAX_DATA counter) */ - if (stream->sendstate.size_inflight < off + len) { - if (stream->stream_id >= 0) - stream->conn->egress.max_data.sent += off + len - stream->sendstate.size_inflight; - stream->sendstate.size_inflight = off + len; - } - if ((ret = quicly_ranges_subtract(&stream->sendstate.pending, off, off + len + is_fin)) != 0) - return ret; - if (wrote_all) { - if ((ret = quicly_ranges_subtract(&stream->sendstate.pending, stream->sendstate.size_inflight, UINT64_MAX)) != 0) - return ret; + /* STREAM frame; if the generated payload extends beyond the end of the current datagram, + * 1. Scatter the output to the payload position of the following datagrams for which the packet headers are yet to be + * generated, as well as having the STREAM frame headers generated. + * 2. Repeatedly commit the packets until we reach the last one, which might not be full-sized. */ + uint16_t scattered_payload_lengths[STREAM_SEND_MAX_EXTRA_DATAGRAMS + 1]; + size_t capacity_of_first_packet = s->dst_end - dst; + dst = scatter_stream_payload(s, stream->conn->egress.max_udp_payload_size, stream->stream_id, off, dst, &len, &wrote_all, + scattered_payload_lengths); + uint64_t off_of_packet = off; + if (scattered_payload_lengths[0] > 0) { + s->dst = s->dst_end; + commit_stream_frame(stream, sent, off_of_packet, s->dst - capacity_of_first_packet, capacity_of_first_packet, 0, 0); + off_of_packet += capacity_of_first_packet; + for (size_t i = 0; scattered_payload_lengths[i + 1] != 0; ++i) { + if ((ret = allocate_ack_eliciting_frame(stream->conn, s, 1, &sent, on_ack_stream)) != 0) { + len = off_of_packet - off; + wrote_all = 0; + is_fin = 0; + goto UpdateStreamState; + } + assert(s->dst == s->dst_payload_from && "scatter does not expect other frames"); + s->dst = s->dst_end; + commit_stream_frame(stream, sent, off_of_packet, s->dst - scattered_payload_lengths[i], + scattered_payload_lengths[i], 0, 0); + off_of_packet += scattered_payload_lengths[i]; + } + if ((ret = allocate_ack_eliciting_frame(stream->conn, s, 1, &sent, on_ack_stream)) != 0) { + len = off_of_packet - off; + wrote_all = 0; + is_fin = 0; + goto UpdateStreamState; + } + assert(s->dst == s->dst_payload_from && "scatter does not expect other frames"); + } + /* determine if the last STREAM frame incorporates FIN, set flags as necessary */ + if (off + len == stream->sendstate.final_size) { + assert(!quicly_sendstate_is_open(&stream->sendstate)); + is_fin = 1; + for (; *s->dst == QUICLY_FRAME_TYPE_PADDING; ++s->dst) + ; + assert((*s->dst & ~QUICLY_FRAME_TYPE_STREAM_BITS) == QUICLY_FRAME_TYPE_STREAM_BASE); + *s->dst |= QUICLY_FRAME_TYPE_STREAM_BIT_FIN; + } else { + is_fin = 0; + } + /* commit the last STREAM frame (without committing the packet, as there could be space left) */ + s->dst = dst; + size_t data_len = (off + len) - off_of_packet; + commit_stream_frame(stream, sent, off_of_packet, s->dst - data_len, data_len, wrote_all, is_fin); } - /* setup sentmap */ - sent->data.stream.stream_id = stream->stream_id; - sent->data.stream.args.start = off; - sent->data.stream.args.end = off + len + is_fin; +UpdateStreamState: + /* update stream sendstate */ + update_stream_sendstate(stream, off, len, is_fin, wrote_all); - return 0; + return ret; } static inline quicly_error_t init_acks_iter(quicly_conn_t *conn, quicly_sentmap_iter_t *iter) @@ -5140,7 +5305,7 @@ static quicly_error_t send_handshake_flow(quicly_conn_t *conn, size_t epoch, qui /* send probe if requested */ if (send_probe) { - if ((ret = do_allocate_frame(conn, s, 1, ALLOCATE_FRAME_TYPE_ACK_ELICITING)) != 0) + if ((ret = allocate_ack_eliciting_frame(conn, s, 1, NULL, NULL)) != 0) goto Exit; *s->dst++ = QUICLY_FRAME_TYPE_PING; conn->egress.last_retransmittable_sent_at = conn->stash.now; @@ -5180,8 +5345,7 @@ static quicly_error_t send_connection_close(quicly_conn_t *conn, size_t epoch, q } /* write frame */ - if ((ret = do_allocate_frame(conn, s, quicly_close_frame_capacity(error_code, offending_frame_type, reason_phrase), - ALLOCATE_FRAME_TYPE_NON_ACK_ELICITING)) != 0) + if ((ret = allocate_frame(conn, s, quicly_close_frame_capacity(error_code, offending_frame_type, reason_phrase), 0)) != 0) return ret; s->dst = quicly_encode_close_frame(s->dst, error_code, offending_frame_type, reason_phrase); @@ -5258,7 +5422,7 @@ static quicly_error_t send_path_challenge(quicly_conn_t *conn, quicly_send_conte { quicly_error_t ret; - if ((ret = do_allocate_frame(conn, s, QUICLY_PATH_CHALLENGE_FRAME_CAPACITY, ALLOCATE_FRAME_TYPE_NON_ACK_ELICITING)) != 0) + if ((ret = allocate_frame(conn, s, QUICLY_PATH_CHALLENGE_FRAME_CAPACITY, 0)) != 0) return ret; s->dst = quicly_encode_path_challenge_frame(s->dst, is_response, data); @@ -5588,8 +5752,9 @@ static quicly_error_t do_send(quicly_conn_t *conn, quicly_send_context_t *s) for (size_t i = 0; i != conn->egress.datagram_frame_payloads.count; ++i) { ptls_iovec_t *payload = conn->egress.datagram_frame_payloads.payloads + i; size_t required_space = quicly_datagram_frame_capacity(*payload); - if ((ret = do_allocate_frame(conn, s, required_space, ALLOCATE_FRAME_TYPE_ACK_ELICITING_NO_CC)) != 0) + if ((ret = allocate_frame(conn, s, required_space, ALLOCATE_FRAME_FLAG_ADJUST_ACK_FREQUENCY)) != 0) goto Exit; + mark_frame_built_as_ack_eliciting(conn, s); if (s->dst_end - s->dst >= required_space) { s->dst = quicly_encode_datagram_frame(s->dst, *payload); QUICLY_PROBE(DATAGRAM_SEND, conn, conn->stash.now, payload->base, payload->len); @@ -5605,7 +5770,7 @@ static quicly_error_t do_send(quicly_conn_t *conn, quicly_send_context_t *s) if (!ack_only) { /* PTO or loss detection timeout, always send PING. This is the easiest thing to do in terms of timer control. */ if (min_packets_to_send != 0) { - if ((ret = do_allocate_frame(conn, s, 1, ALLOCATE_FRAME_TYPE_ACK_ELICITING)) != 0) + if ((ret = allocate_ack_eliciting_frame(conn, s, 1, NULL, NULL)) != 0) goto Exit; if (get_epoch(s->current.first_byte) == QUICLY_EPOCH_1RTT && conn->super.remote.transport_params.min_ack_delay_usec != UINT64_MAX) { @@ -5721,6 +5886,10 @@ static quicly_error_t do_send(quicly_conn_t *conn, quicly_send_context_t *s) } } if (ret == 0 && s->target.first_byte_at != NULL) { + /* If only a STREAM frame was to be built but `on_send_emit` returned BLOCKED, we might have built zero frames. Assuming + * that it is rare to see BLOCKED, send a PADDING-only packet (TODO skip sending the packet at all) */ + if (s->dst == s->dst_payload_from) + *s->dst++ = QUICLY_FRAME_TYPE_PADDING; /* last packet can be small-sized, unless it is the first flight sent from the client */ if ((s->payload_buf.datagram[0] & QUICLY_PACKET_TYPE_BITMASK) == QUICLY_PACKET_TYPE_INITIAL && (quicly_is_client(conn) || !ack_only)) diff --git a/t/lossy.c b/t/lossy.c index 312e3daf..cb999de6 100644 --- a/t/lossy.c +++ b/t/lossy.c @@ -425,21 +425,21 @@ static void test_downstream(void) subtest("75%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 3, 5131, 15505, 2644, 3280, 12824); + loss_check_stats(time_spent, 4, 5131, 20832, 2636, 3280, 13054); for (i = 0; i != 100; ++i) { init_cond_rand(&loss_cond_down, 1, 2); subtest("50%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 0, 792, 1104, 484, 484, 1526); + loss_check_stats(time_spent, 0, 792, 3261, 484, 488, 1942); for (i = 0; i != 100; ++i) { init_cond_rand(&loss_cond_down, 1, 4); subtest("25%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 0, 228.7, 244, 230, 230, 408); + loss_check_stats(time_spent, 0, 216, 255, 210, 230, 472); for (i = 0; i != 100; ++i) { init_cond_rand(&loss_cond_down, 1, 10); @@ -453,7 +453,7 @@ static void test_downstream(void) subtest("5%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 0, 103, 118, 80, 80, 230); + loss_check_stats(time_spent, 0, 96, 118, 80, 80, 230); for (i = 0; i != 100; ++i) { init_cond_rand(&loss_cond_down, 1, 40); @@ -467,7 +467,7 @@ static void test_downstream(void) subtest("1.6%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 0, 86, 91.1, 80, 80, 80); + loss_check_stats(time_spent, 0, 83, 91.1, 80, 80, 80); } static void test_bidirectional(void) @@ -482,7 +482,7 @@ static void test_bidirectional(void) subtest("75%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 18, 203900, 223090, 65015, 84162, 633710); + loss_check_stats(time_spent, 19, 203900, 227342, 64380, 84162, 633710); for (i = 0; i != 100; ++i) { init_cond_rand(&loss_cond_down, 1, 2); @@ -490,7 +490,7 @@ static void test_bidirectional(void) subtest("50%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 0, 4612, 5610, 886, 1175, 9269); + loss_check_stats(time_spent, 0, 4612, 6600, 886, 1175, 9269); for (i = 0; i != 100; ++i) { init_cond_rand(&loss_cond_down, 1, 4); @@ -498,7 +498,7 @@ static void test_bidirectional(void) subtest("25%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 0, 270, 280, 257, 284, 478); + loss_check_stats(time_spent, 0, 269, 360, 230, 284, 652); for (i = 0; i != 100; ++i) { init_cond_rand(&loss_cond_down, 1, 10); @@ -506,7 +506,7 @@ static void test_bidirectional(void) subtest("10%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 0, 121, 137, 80, 80, 230); + loss_check_stats(time_spent, 0, 121, 162, 80, 80, 298); for (i = 0; i != 100; ++i) { init_cond_rand(&loss_cond_down, 1, 20); @@ -514,7 +514,7 @@ static void test_bidirectional(void) subtest("5%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 0, 101, 109, 80, 80, 190); + loss_check_stats(time_spent, 0, 101, 117, 80, 80, 230); for (i = 0; i != 100; ++i) { init_cond_rand(&loss_cond_down, 1, 40); @@ -522,7 +522,7 @@ static void test_bidirectional(void) subtest("2.5%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 0, 95, 97, 80, 80, 190); + loss_check_stats(time_spent, 0, 95, 113, 80, 80, 190); for (i = 0; i != 100; ++i) { init_cond_rand(&loss_cond_down, 1, 64); @@ -530,7 +530,7 @@ static void test_bidirectional(void) subtest("1.6%", loss_core); time_spent[i] = quic_now - 1; } - loss_check_stats(time_spent, 0, 86, 91, 80, 80, 80); + loss_check_stats(time_spent, 0, 86, 101, 80, 80, 190); } void test_lossy(void) diff --git a/t/test.c b/t/test.c index 069d8557..589ec93b 100644 --- a/t/test.c +++ b/t/test.c @@ -155,6 +155,42 @@ static void test_error_codes(void) ok(!QUICLY_ERROR_IS_QUIC_APPLICATION(a)); } +static void test_adjust_crypto_frame_layout(void) +{ +#define TEST(_capacity, check) \ + do { \ + uint8_t buf[] = {0x06, 0x04, 'h', 'e', 'l', 'l', 'o', 0, 0, 0}; \ + uint8_t *dst = buf + 2, *const dst_end = buf + _capacity; \ + size_t len = 5; \ + int wrote_all = 1; \ + dst = adjust_crypto_frame_layout(dst, dst_end, &len, &wrote_all); \ + do { \ + check \ + } while (0); \ + } while (0); + + /* test CRYPTO frames that fit and don't when length is inserted */ + TEST(10, { + ok(dst == buf + 8); + ok(len == 5); + ok(wrote_all); + ok(memcmp(buf, "\x06\x04\x05hello", 8) == 0); + }); + TEST(18, { + ok(dst == buf + 8); + ok(len == 5); + ok(wrote_all); + ok(memcmp(buf, "\x06\x04\x05hello", 8) == 0); + }); + TEST(7, { + ok(dst == buf + 7); + ok(len == 4); + ok(!wrote_all); + ok(memcmp(buf, "\x06\x04\x04hell", 7) == 0); + }); +#undef TEST +} + static uint16_t test_enable_with_ratio255_random_value; static void test_enable_with_ratio255_get_random(void *p, size_t len) @@ -181,65 +217,146 @@ static void test_enable_with_ratio255(void) ok(num_enabled == 63 * (65535 / 255)); } -static void test_adjust_stream_frame_layout(void) +static void test_adjust_last_stream_frame(void) { -#define TEST(_is_crypto, _capacity, check) \ +#define TEST(space_left, check) \ do { \ - uint8_t buf[] = {0xff, 0x04, 'h', 'e', 'l', 'l', 'o', 0, 0, 0}; \ - uint8_t *dst = buf + 2, *const dst_end = buf + _capacity, *frame_at = buf; \ - size_t len = 5; \ - int wrote_all = 1; \ - buf[0] = _is_crypto ? 0x06 : 0x08; \ - adjust_stream_frame_layout(&dst, dst_end, &len, &wrote_all, &frame_at); \ + uint8_t buf[] = {0x08, 0x04, 'h', 'e', 'l', 'l', 'o', 0, 0, 0}; \ + size_t increase = adjust_last_stream_frame(buf, 2, 5, space_left, 1); \ do { \ check \ } while (0); \ - } while (0); + } while (0) - /* test CRYPTO frames that fit and don't when length is inserted */ - TEST(1, 10, { - ok(dst == buf + 8); - ok(len == 5); - ok(wrote_all); - ok(frame_at == buf); - ok(memcmp(buf, "\x06\x04\x05hello", 8) == 0); + /* test STREAM frames */ + TEST(0, { + ok(increase == 0); + ok(memcmp(buf, "\x08\x04hello", 7) == 0); }); - TEST(1, 8, { - ok(dst == buf + 8); - ok(len == 5); - ok(wrote_all); - ok(frame_at == buf); - ok(memcmp(buf, "\x06\x04\x05hello", 8) == 0); + TEST(1, { + ok(increase == 1); + ok(memcmp(buf, "\x00\x08\x04hello", 7) == 0); }); - TEST(1, 7, { - ok(dst == buf + 7); - ok(len == 4); - ok(!wrote_all); - ok(frame_at == buf); - ok(memcmp(buf, "\x06\x04\x04hell", 7) == 0); + TEST(2, { + ok(increase == 1); + ok(memcmp(buf, "\x0a\x04\x05hello", 7) == 0); }); - /* test STREAM frames */ - TEST(0, 9, { - ok(dst == buf + 8); - ok(len == 5); - ok(wrote_all); - ok(frame_at == buf); - ok(memcmp(buf, "\x0a\x04\x05hello", 8) == 0); - }); - TEST(0, 8, { - ok(dst == buf + 8); - ok(len == 5); - ok(wrote_all); - ok(frame_at == buf + 1); - ok(memcmp(buf, "\x00\x08\x04hello", 8) == 0); +#undef TEST +} + +static void test_scatter_stream_payload(void) +{ + quicly_cid_t dcid = {.cid = {'C', 'I', 'D'}, .len = 3}; + ptls_aead_context_t aead = {.algo = &ptls_openssl_aes128gcm}; + struct st_quicly_cipher_context_t cipher = {.aead = &aead}; + +#define TEST(_len, datagram_size, check) \ + do { \ + uint8_t buf[] = "\x08\x04" \ + "Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to " \ + "do: once or twice she had peeped into the book her sister was reading, but it had no pictures or " \ + "conversations in it, `and what is the use of a book,' thought Alice `without pictures or conversation?'" \ + "\n\nSo she was considering in her own mind (as well as she could, for the hot day made her feel very " \ + "sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting " \ + "and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her."; \ + quicly_send_context_t s = { \ + .dcid = &dcid, \ + .current.cipher = &cipher, \ + .dst = buf, \ + .dst_end = buf + 8, \ + }; \ + size_t len = (_len); \ + int wrote_all = 1; \ + uint16_t scattered_payload_lengths[11]; \ + memset(scattered_payload_lengths, 0x55, sizeof(scattered_payload_lengths)); \ + uint8_t *end_of_last_frame = \ + scatter_stream_payload(&s, datagram_size, 4, 0, buf + 2, &len, &wrote_all, scattered_payload_lengths); \ + do { \ + check \ + } while (0); \ + } while (0) + + /* test the case where all space are used */ + TEST(150 /* 6 (current) + 16 * 9 incl. some extra */, 38 /* 16 bytes frame space per datagram */, { + ok(len == 119); + ok(wrote_all == 0); + ok(scattered_payload_lengths[0] == 13); + ok(scattered_payload_lengths[1] == 13); + ok(scattered_payload_lengths[2] == 13); + ok(scattered_payload_lengths[3] == 13); + ok(scattered_payload_lengths[4] == 13); + ok(scattered_payload_lengths[5] == 12); + ok(scattered_payload_lengths[6] == 12); + ok(scattered_payload_lengths[7] == 12); + ok(scattered_payload_lengths[8] == 12); + ok(scattered_payload_lengths[9] == 0); + ok(memcmp(buf, + "\x08\x04" + "Alice ", + 8) == 0); + size_t payload_gap = aead.algo->tag_size + 1 + dcid.len + QUICLY_SEND_PN_SIZE; + ok(memcmp(buf + 8 + payload_gap, + "\x0c\x04\x06" + "was beginning", + 16) == 0); + ok(memcmp(buf + 24 + payload_gap * 2, + "\x0c\x04\x13" + " to get very ", + 16) == 0); + ok(memcmp(buf + 40 + payload_gap * 3, + "\x0c\x04\x20" + "tired of sitt", + 16) == 0); + ok(memcmp(buf + 56 + payload_gap * 4, + "\x0c\x04\x2d" + "ing by her si", + 16) == 0); + ok(memcmp(buf + 72 + payload_gap * 5, + "\x0c\x04\x3a" + "ster on the b", + 16) == 0); + ok(memcmp(buf + 88 + payload_gap * 6, + "\x0c\x04\x40\x47" + "ank, and of ", + 16) == 0); + ok(memcmp(buf + 104 + payload_gap * 7, + "\x0c\x04\x40\x53" + "having nothi", + 16) == 0); + ok(memcmp(buf + 120 + payload_gap * 8, + "\x0c\x04\x40\x5f" + "ng to do: on", + 16) == 0); + ok(buf + 8 + 38 * 9 == end_of_last_frame); }); - TEST(0, 7, { - ok(dst == buf + 7); - ok(len == 5); - ok(wrote_all); - ok(frame_at == buf); - ok(memcmp(buf, "\x08\x04hello", 7) == 0); + + /* test the case where some space left */ + TEST(34 /* 6 (current) + 13 * 2 + 2 */, 38 /* 16 bytes frame space per datagram */, { + ok(len == 34); + ok(wrote_all == 1); + ok(scattered_payload_lengths[0] == 13); + ok(scattered_payload_lengths[1] == 13); + ok(scattered_payload_lengths[2] == 2); + ok(scattered_payload_lengths[3] == 0); + ok(memcmp(buf, + "\x08\x04" + "Alice ", + 8) == 0); + size_t payload_gap = aead.algo->tag_size + 1 + dcid.len + QUICLY_SEND_PN_SIZE; + ok(memcmp(buf + 8 + payload_gap, + "\x0c\x04\x06" + "was beginning", + 16) == 0); + ok(memcmp(buf + 24 + payload_gap * 2, + "\x0c\x04\x13" + " to get very ", + 16) == 0); + ok(memcmp(buf + 40 + payload_gap * 3, + "\x0e\x04\x20\x02" + "ti", + 6) == 0); + ok(buf + 40 + payload_gap * 3 + 6 == end_of_last_frame); }); #undef TEST @@ -285,6 +402,8 @@ const quicly_cid_plaintext_t *new_master_id(void) return &master; } +static int use_scatter_emit; + static quicly_error_t on_stream_open(quicly_stream_open_t *self, quicly_stream_t *stream) { test_streambuf_t *sbuf; @@ -297,6 +416,8 @@ static quicly_error_t on_stream_open(quicly_stream_open_t *self, quicly_stream_t sbuf->error_received.reset_stream = -1; stream->callbacks = &stream_callbacks; + stream->scatter_emit = use_scatter_emit; + return 0; } @@ -1288,7 +1409,8 @@ static void test_state_exhaustion(void) /* send up to 200 packets with stream frame having gaps and check that the receiver raises state exhaustion */ for (size_t i = 0; i < 200; ++i) { test_setup_send_context(client, &s, &datagram, buf, sizeof(buf)); - do_allocate_frame(client, &s, 100, ALLOCATE_FRAME_TYPE_ACK_ELICITING); + allocate_frame(client, &s, 100, ALLOCATE_FRAME_FLAG_CONSULT_CC | ALLOCATE_FRAME_FLAG_ADJUST_ACK_FREQUENCY); + mark_frame_built_as_ack_eliciting(client, &s); *s.dst++ = QUICLY_FRAME_TYPE_STREAM_BASE | QUICLY_FRAME_TYPE_STREAM_BIT_OFF | QUICLY_FRAME_TYPE_STREAM_BIT_LEN; s.dst = quicly_encodev(s.dst, 0); /* stream id */ s.dst = quicly_encodev(s.dst, i * 2); /* off */ @@ -1461,6 +1583,26 @@ static void test_stats_foreach(void) #undef CHECK } +static void test_endpoints(int input_flag) +{ + int use_scatter_emit_backup = use_scatter_emit; + use_scatter_emit = input_flag; + + subtest("simple", test_simple); + subtest("stream-concurrency", test_stream_concurrency); + subtest("lossy", test_lossy); + subtest("test-nondecryptable-initial", test_nondecryptable_initial); + subtest("set_cc", test_set_cc); + subtest("ecn-index-from-bits", test_ecn_index_from_bits); + subtest("jumpstart-cwnd", test_jumpstart_cwnd); + subtest("jumpstart", test_jumpstart); + subtest("cc", test_cc); + subtest("state-exhaustion", test_state_exhaustion); + subtest("migration-during-handshake", test_migration_during_handshake); + + use_scatter_emit = use_scatter_emit_backup; +} + int main(int argc, char **argv) { static ptls_iovec_t cert; @@ -1508,6 +1650,7 @@ int main(int argc, char **argv) quicly_amend_ptls_context(quic_ctx.tls); + /* module-level tests */ subtest("ack_frequency_handling", test_ack_frequency); subtest("error-codes", test_error_codes); subtest("enable_with_ratio255", test_enable_with_ratio255); @@ -1521,25 +1664,17 @@ int main(int argc, char **argv) subtest("pacer", test_pacer); subtest("sentmap", test_sentmap); subtest("loss", test_loss); - subtest("adjust-stream-frame-layout", test_adjust_stream_frame_layout); + subtest("adjust-crypto-frame-layout", test_adjust_crypto_frame_layout); + subtest("adjust-last-stream-frame", test_adjust_last_stream_frame); + subtest("scatter-stream-payload", test_scatter_stream_payload); subtest("test-vector", test_vector); subtest("test-retry-aead", test_retry_aead); subtest("transport-parameters", test_transport_parameters); subtest("cid", test_cid); - subtest("simple", test_simple); - subtest("stream-concurrency", test_stream_concurrency); - subtest("lossy", test_lossy); - subtest("test-nondecryptable-initial", test_nondecryptable_initial); - subtest("set_cc", test_set_cc); - subtest("ecn-index-from-bits", test_ecn_index_from_bits); - subtest("jumpstart-cwnd", test_jumpstart_cwnd); - subtest("jumpstart", test_jumpstart); - subtest("cc", test_cc); - - subtest("state-exhaustion", test_state_exhaustion); - subtest("migration-during-handshake", test_migration_during_handshake); - subtest("stats-foreach", test_stats_foreach); + subtest("test-endpoints", test_endpoints, 0); + subtest("test-endpoints-scattering", test_endpoints, 1); + return done_testing(); }