rfc9693xml2.original.xml   rfc9693.xml 
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="utf-8"?>
<!-- This template is for creating an Internet Draft using xml2rfc,
which is available here: http://xml.resource.org. -->
<!DOCTYPE rfc SYSTEM "rfc2629.dtd" [
<!-- One method to get references from the online citation libraries.
There has to be one entity for each item to be referenced.
An alternate method (rfc include) is described in the references. -->
<!ENTITY RFC1918 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC <!DOCTYPE rfc [
.1918.xml"> <!ENTITY nbsp "&#160;">
<!ENTITY RFC2119 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC <!ENTITY zwsp "&#8203;">
.2119.xml"> <!ENTITY nbhy "&#8209;">
<!ENTITY RFC2544 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC <!ENTITY wj "&#8288;">
.2544.xml">
<!ENTITY RFC3022 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.3022.xml">
<!ENTITY RFC4340 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.4340.xml">
<!ENTITY RFC4814 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.4814.xml">
<!ENTITY RFC5180 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.5180.xml">
<!ENTITY RFC6146 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.6146.xml">
<!ENTITY RFC7599 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.7599.xml">
<!ENTITY RFC8174 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.8174.xml">
<!ENTITY RFC8219 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.8219.xml">
<!ENTITY RFC9000 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.9000.xml">
<!ENTITY RFC9260 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.9260.xml">
<!ENTITY RFC9411 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.9411.xml">
]> ]>
<?xml-stylesheet type='text/xsl' href='rfc2629.xslt' ?>
<!-- used by XSLT processors --> <rfc xmlns:xi="http://www.w3.org/2001/XInclude" category="info" docName="draft-i
<!-- For a complete list and description of processing instructions (PIs), etf-bmwg-benchmarking-stateful-09" number="9693" consensus="true" ipr="trust2009
please see http://xml.resource.org/authoring/README.html. --> 02" obsoletes="" updates="" submissionType="IETF" xml:lang="en" tocInclude="true
<!-- Below are generally applicable Processing Instructions (PIs) that most I-Ds " tocDepth="4" symRefs="true" sortRefs="true" version="3">
might want to use.
(Here they are set differently than their defaults in xml2rfc v1.32) -->
<?rfc strict="yes" ?>
<!-- give errors regarding ID-nits and DTD validation -->
<!-- control the table of contents (ToC) -->
<?rfc toc="yes"?>
<!-- generate a ToC -->
<?rfc tocdepth="4"?>
<!-- the number of levels of subsections in ToC. default: 3 -->
<!-- control references -->
<?rfc symrefs="yes"?>
<!-- use symbolic references tags, i.e, [RFC2119] instead of [1] -->
<?rfc sortrefs="yes" ?>
<!-- sort the reference entries alphabetically -->
<!-- control vertical white space
(using these PIs as follows is recommended by the RFC Editor) -->
<?rfc compact="yes" ?>
<!-- do not start each main section on a new page -->
<?rfc subcompact="no" ?>
<!-- keep one blank line between list items -->
<!-- end of list of popular I-D processing instructions -->
<rfc category="info" docName="draft-ietf-bmwg-benchmarking-stateful-09" ipr="tru
st200902">
<front> <front>
<!-- The abbreviated title is used in the page header - it is only necessary
if the
full title is longer than 39 characters -->
<title abbrev="Benchmarking Stateful NATxy Gateways">Benchmarking Methodolog <!-- [rfced] As RFC 4814 is mentioned in this document's Abstract and
y Introduction, may we remove the reference to it from the title?
for Stateful NATxy Gateways using RFC 4814 Pseudorandom Port Numbers</title>
<!-- add 'role="editor"' below for the editors if appropriate --> Original:
<!-- Another author who claims to be an editor --> Benchmarking Methodology for Stateful NATxy Gateways using RFC 4814
Pseudorandom Port Numbers
Perhaps:
Benchmarking Methodology for Stateful NATxy Gateways Using
Pseudorandom Port Numbers
-->
<title abbrev="Benchmarking Stateful NATxy Gateways">Benchmarking
Methodology for Stateful NATxy Gateways Using RFC 4814 Pseudorandom Port
Numbers</title>
<seriesInfo name="RFC" value="9693"/>
<author fullname="Gábor Lencse" initials="G." surname="Lencse"> <author fullname="Gábor Lencse" initials="G." surname="Lencse">
<organization>Széchenyi István University</organization> <organization>Széchenyi István University</organization>
<address> <address>
<postal> <postal>
<street>Egyetem tér 1.</street> <street>Egyetem tér 1.</street>
<!-- Reorder these if your country does things differently -->
<city>Győr</city> <city>Győr</city>
<region></region>
<code>H-9026</code> <code>H-9026</code>
<country>Hungary</country> <country>Hungary</country>
</postal> </postal>
<phone></phone>
<email>lencse@sze.hu</email> <email>lencse@sze.hu</email>
<uri></uri>
</address> </address>
</author> </author>
<author fullname="Keiichi Shima" initials="K." surname="Shima"> <author fullname="Keiichi Shima" initials="K." surname="Shima">
<organization>SoftBank Corp.</organization> <organization>SoftBank Corp.</organization>
<address> <address>
<postal> <postal>
<street>1-7-1 Kaigan</street> <street>1-7-1 Kaigan</street>
<city>Minato-ku</city> <region>Minato-ku, Tokyo</region>
<region>Tokyo</region>
<code>105-7529</code> <code>105-7529</code>
<country>Japan</country> <country>Japan</country>
</postal> </postal>
<phone></phone>
<email>shima@wide.ad.jp</email> <email>shima@wide.ad.jp</email>
<uri>https://softbank.co.jp/</uri> <uri>https://softbank.co.jp/</uri>
</address> </address>
</author> </author>
<date year="2024" month="December"/>
<date year="2024" /> <area>OPS</area>
<workgroup>bmwg</workgroup>
<!-- Meta-data Declarations --> <keyword>Benchmarking</keyword>
<keyword>Stateful NATxy</keyword>
<keyword>Measurement Procedure</keyword>
<keyword>Throughput</keyword>
<keyword>Frame Loss Rate</keyword>
<keyword>Latency</keyword>
<keyword>PDV</keyword>
<area>Operations and Management Area</area> <abstract>
<t>RFC 2544 defines a benchmarking methodology for network
interconnect devices. RFC 5180 addresses IPv6 specificities, and it also
provides a technology update but excludes IPv6 transition technologies.
RFC 8219 addresses IPv6 transition technologies, including stateful
NAT64. However, none of them discuss how to apply pseudorandom port
numbers from RFC 4814 to any stateful NATxy (such as NAT44, NAT64, and NAT
66)
technologies. This document discusses why using pseudorandom port
numbers with stateful NATxy gateways is a difficult problem. It
recommends a solution that limits the port number ranges and uses two
test phases (phase 1 and phase 2). This document shows how the classic
performance measurement procedures (e.g., throughput, frame loss rate,
latency, etc.) can be carried out. New performance metrics and
measurement procedures are also defined for measuring the maximum
connection establishment rate, connection tear-down rate, and
connection tracking table capacity.
</t>
</abstract>
</front>
<workgroup>Benchmarking Methodology Working Group</workgroup> <middle>
<section anchor="intro" numbered="true" toc="default">
<name>Introduction</name>
<!-- WG name at the upperleft corner of the doc, <t><xref target="RFC2544" format="default"/> defines a comprehensive
IETF is fine for individual submissions. benchmarking methodology for network interconnect devices that is still
If this element is not present, the default is "Network Working Group", in use. It is mainly indpendent of IP version, but it uses IPv4 in its
which is used by the RFC Editor as a nod to the history of the IETF. - examples. <xref target="RFC5180" format="default"/> addresses IPv6
-> specificities and also adds technology updates but declares IPv6
transition technologies are out of its scope. <xref target="RFC8219"
format="default"/> addresses the IPv6 transition technologies, including
stateful NAT64. It reuses several benchmarking procedures from <xref
target="RFC2544" format="default"/> (e.g., throughput, frame loss rate),
and it redefines the latency measurement and adds further ones (e.g., the
Packet Delay Variation (PDV) measurement).</t>
<keyword>Benchmarking, Stateful NATxy, Measurement Procedure, Throughput, Fr <!-- [rfced] Per Section 3.6 of RFC 7322 ("RFC Style Guide"), abbreviations
ame Loss Rate, Latency, PDV</keyword> must be expanded upon first use. To avoid expanding "NAPT" upon first use
here and stacking multiple sets of parentheses, we have rephrased as
follows (because "NAPT" is introduced and expanded later in this document).
Please let us know of any objections.
<!-- Keywords will be incorporated into HTML output Original:
files in a meta tag but they have no effect on text or nroff
output. If you submit your draft to the RFC Editor, the
keywords will be used for the search engine. -->
<abstract> However, none of them discussed, how to apply [RFC4814] pseudorandom
<t>RFC 2544 has defined a benchmarking methodology for network port numbers, when benchmarking stateful NATxy (NAT44 (also called
interconnect devices. RFC 5180 addressed IPv6 specificities and it also NAPT) [RFC3022], NAT64 [RFC6146], and NAT66) gateways. (It should be
provided a technology update but excluded IPv6 transition technologies. noted that stateful NAT66 is not an IETF specification but refers to
RFC 8219 addressed IPv6 transition technologies, including stateful NAT64. an IPv6 version of the stateful NAT44 specification.)
However, none of them discussed how to apply RFC 4814 pseudorandom port
numbers
to any stateful NATxy (NAT44, NAT64, NAT66) technologies.
This document discusses why using pseudorandom port numbers with statef
ul NATxy gateways is a
difficult problem. It recommends a solution limiting the port number ra
nges and using
two test phases (phase 1 and phase 2). It is shown how the classic perf
ormance measurement
procedures (e.g. throughput, frame loss rate, latency, etc.) can be car
ried out.
New performance metrics and measurement procedures are also defined for
measuring
maximum connection establishment rate, connection tear-down rate, and c
onnection
tracking table capacity.
</t>
</abstract>
</front>
<middle> Current:
<section anchor="intro" title="Introduction">
<t><xref target="RFC2544"/> has defined a comprehensive benchmarking
methodology for network interconnect devices, which is still in use. It
was
mainly IP version independent, but it used IPv4 in its examples.
<xref target="RFC5180"/> addressed IPv6 specificities
and also added technology updates, but declared IPv6 transition technologi
es
out of its scope. <xref target="RFC8219"/> addressed the IPv6 transition
technologies, including stateful NAT64. It has reused several benchmark
ing
procedures from <xref target="RFC2544"/> (e.g. throughput, frame loss rate
),
it has redefined the latency measurement and added further ones, e.g. t
he PDV
(packet delay variation) measurement.</t>
<t>However, none of them discussed, how to apply <xref target="RFC4814"
/>
pseudorandom port numbers, when benchmarking stateful NATxy (NAT44 (als
o called
NAPT) <xref target="RFC3022"/>, NAT64 <xref target="RFC6146"/>, and NAT
66) gateways.
(It should be noted that stateful NAT66 is not an IETF specification bu
t
refers to an IPv6 version of the stateful NAT44 specification.) The aut
hors are not
aware of any other RFCs that address this question.
</t>
<t>First, it is discussed why using pseudorandom port numbers with statefu
l NATxy
gateways is a difficult problem.
</t>
<t>Then a solution is recommended.
</t>
<section> However, none of them discussed how to apply pseudorandom port numbers from
[RFC4814] when benchmarking stateful NATxy gateways (such as NAT44
[RFC3022], NAT64 [RFC6146], and NAT66). (It should be
noted that stateful NAT66 is not an IETF specification but refers to
an IPv6 version of the stateful NAT44 specification.)
-->
<t>However, none of them discuss how to apply pseudorandom port
numbers from <xref target="RFC4814" format="default"/> when benchmarking
stateful NATxy gateways (such as NAT44 <xref
target="RFC3022" format="default"/>, NAT64 <xref target="RFC6146"
format="default"/>, and NAT66). (It should be noted that stateful NAT66
is not an IETF specification but refers to an IPv6 version of the
stateful NAT44 specification.) The authors are not aware of any other
RFCs that address this question.
</t>
<t>First, this document discusses why using pseudorandom port numbers with
stateful NATxy gateways is a difficult problem. Then, a solution is
recommended.</t>
<section numbered="true" toc="default">
<name>Requirements Language</name> <name>Requirements Language</name>
<t>The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", <t>The key words "<bcp14>MUST</bcp14>", "<bcp14>MUST NOT</bcp14>",
"SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT "<bcp14>REQUIRED</bcp14>", "<bcp14>SHALL</bcp14>", "<bcp14>SHALL
RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be NOT</bcp14>", "<bcp14>SHOULD</bcp14>", "<bcp14>SHOULD NOT</bcp14>",
interpreted as described in BCP 14 <xref target="RFC2119"/> "<bcp14>RECOMMENDED</bcp14>", "<bcp14>NOT RECOMMENDED</bcp14>",
<xref target="RFC8174"/> when, and only when, they appear in "<bcp14>MAY</bcp14>", and "<bcp14>OPTIONAL</bcp14>" in this document
all capitals, as shown here.</t> are to be interpreted as described in BCP 14 <xref target="RFC2119"
format="default"/> <xref target="RFC8174" format="default"/> when, and
only when, they appear in all capitals, as shown here.</t>
</section> </section>
<!-- [CHECK] The 'Requirements Language' section is optional -->
</section> </section>
<section anchor="problem" numbered="true" toc="default">
<name>Pseudorandom Port Numbers and Stateful Translation</name>
<section anchor="problem" title="Pseudorandom Port Numbers and Stateful Tran <t>In its appendix, <xref target="RFC2544" format="default"/>
slation"> defines a frame format for test frames, including specific source and
<t>In its appendix, <xref target="RFC2544"/> has defined a frame format destination port numbers. <xref target="RFC4814" format="default"/>
for test frames including specific source and destination port numbers. recommends using pseudorandom and uniformly distributed values for both
<xref target="RFC4814"/> recommends using pseudorandom and source and destination port numbers. However, stateful NATxy (such as NAT4
uniformly distributed values for both source and destination port 4,
numbers. However, stateful NATxy (NAT44, NAT64, NAT66) solutions use the NAT64, and NAT66) solutions use the port numbers to identify
port numbers to identify connections. The usage of pseudorandom port connections. The usage of pseudorandom port numbers causes different
numbers causes different problems depending on the direction. problems depending on the direction:
<list style="symbols"> </t>
<t> As for the client-to-server direction, pseudorandom source and <ul spacing="normal">
destination port numbers could be used, however, this approach would <li>
be a denial of service attack against the stateful NATxy gateway, <t>For the client-to-server direction, pseudorandom source and
destination port numbers could be used; however, this approach would
be a denial-of-service attack against the stateful NATxy gateway,
because it would exhaust its connection tracking table capacity. To tha t end, because it would exhaust its connection tracking table capacity. To tha t end,
let us see some calculations using the recommendations of RFC 4814: let us see some calculations using the recommendations of <xref target=
<list style="symbols"> "RFC4814" format="default"/>:
<t>The recommended source port range is: 1024-65535, thus its size is </t>
: 64512.</t> <ul spacing="normal">
<t>The recommended destination port range is: 1-49151, thus its size <li>
is: 49151.</t> <t>The recommended source port range is 1024-65535; thus, its size
<t>The number of source and destination port number combinations is: is 64512.</t>
3,170,829,312.</t> </li>
</list> <li>
<t>The recommended destination port range is 1-49151; thus, its si
ze is 49151.</t>
</li>
<li>
<t>The number of source and destination port number combinations i
s 3,170,829,312.</t>
</li>
</ul>
<t>
It should be noted that the usage of different source and destination IP a ddresses It should be noted that the usage of different source and destination IP a ddresses
further increases the number of connection tracking table entries.</t> further increases the number of connection tracking table entries.</t>
<t>As for the server-to-client direction, the stateful DUT (Device Unde </li>
r Test) would drop any <li>
packets that do not belong to an existing connection, therefore, the <t>For the server-to-client direction, the stateful Device Under Test
direct usage of pseudorandom port numbers from the above-mentioned rang (DUT) would drop any
es packets that do not belong to an existing connection; therefore, the
direct usage of pseudorandom port numbers from the ranges mentioned abo
ve
is not feasible.</t> is not feasible.</t>
</list> </li>
</t> </ul>
</section> </section>
<section anchor="setup_term" title="Test Setup and Terminology"> <section anchor="setup_term" numbered="true" toc="default">
<name>Test Setup and Terminology</name>
<t>Section 12 of <xref target="RFC2544"/> requires testing first using
a single protocol source and destination address pair an then also usin
g
multiple protocol addresses. The same approach is followed: first, a
single source and destination IP address pair is used, and then it is e
xplained how to
use multiple IP addresses.</t>
<section anchor="setup_term_single" title="When Testing with a Single IP A <t><xref target="RFC2544" sectionFormat="of" section="12"/> requires
ddress Pair"> testing using a single protocol source and destination address pair
first and then also using multiple protocol addresses. The same
approach is followed: first, a single source and destination IP address
pair is used, and then it is explained how to use multiple IP
addresses.</t>
<t>The methodology works with any IP versions to benchmark stateful N <section anchor="setup_term_single" numbered="true" toc="default">
ATxy <name>When Testing with a Single IP Address Pair</name>
gateways, where x and y are in {4, 6}. To facilitate an easy unde
rstanding,
two typical examples are used: stateful NAT44 and stateful NAT64.
</t>
<t>The Test Setup for the well-known stateful NAT44 (also called NAPT <t>The methodology works with any IP version to benchmark stateful
: NATxy gateways, where x and y are in {4, 6}. To facilitate an easy
Network Address and Port Translation) solution is shown in understanding, two typical examples are used: stateful NAT44 and
<xref target="test_setup_sfnat44"/>.</t> stateful NAT64.</t>
<t>Note that the <xref target="RFC1918"/> private IP addresses are <t>The test setup for the well-known stateful NAT44 (also called
used to facilitate an easy understanding of the example. And the Network Address and Port Translation (NAPT)) solution is shown in
usage of the IP addresses reserved for benchmarking is absolutely <xref target="test_setup_sfnat44" format="default"/>.</t>
legitimate.</t>
<figure anchor="test_setup_sfnat44" align="center" title="Test setup for <t>Note that the private IP addresses from <xref target="RFC1918"
benchmarking format="default"/> are used to facilitate an easy understanding of the
stateful NAT44 gateways"> example, and the usage of the IP addresses reserved for benchmarking
<preamble></preamble> is absolutely legitimate.</t>
<artwork align="left"><![CDATA[ <t keepWithNext="true"/>
<figure anchor="test_setup_sfnat44">
<name>Test Setup for Benchmarking Stateful NAT44 Gateways</name>
<artwork align="left" name="" type="" alt=""><![CDATA[
+--------------------------------------+ +--------------------------------------+
10.0.0.2 |Initiator Responder| 198.19.0.2 10.0.0.2 |Initiator Responder| 198.19.0.2
+-------------| Tester |<------------+ +-------------| Tester |<------------+
| private IPv4| [state table]| public IPv4 | | private IPv4| [state table]| public IPv4 |
| +--------------------------------------+ | | +--------------------------------------+ |
| | | |
| +--------------------------------------+ | | +--------------------------------------+ |
| 10.0.0.1 | DUT: | 198.19.0.1 | | 10.0.0.1 | DUT: | 198.19.0.1 |
+------------>| Stateful NAT44 gateway |-------------+ +------------>| Stateful NAT44 gateway |-------------+
private IPv4| [connection tracking table] | public IPv4 private IPv4| [connection tracking table] | public IPv4
+--------------------------------------+ +--------------------------------------+
]]></artwork>
]]></artwork>
<postamble></postamble>
</figure> </figure>
<t keepWithPrevious="true"/>
<t>The test setup for the stateful NAT64 solution <xref target="RFC6146"
format="default"/>, which is also widely used, is shown in
<xref target="test_setup_sfnat64" format="default"/>.</t>
<t> The Test Setup for the also widely used stateful NAT64 <xref targ <t keepWithNext="true"/>
et="RFC6146"/> <figure anchor="test_setup_sfnat64">
solution is shown in <xref target="test_setup_sfnat64"/>.</t> <name>Test Setup for Benchmarking Stateful NAT64 Gateways</name>
<artwork align="left" name="" type="" alt=""><![CDATA[
<figure anchor="test_setup_sfnat64" align="center" title="Test setup for
benchmarking
stateful NAT64 gateways">
<preamble></preamble>
<artwork align="left"><![CDATA[
+--------------------------------------+ +--------------------------------------+
2001:2::2 |Initiator Responder| 198.19.0.2 2001:2::2 |Initiator Responder| 198.19.0.2
+-------------| Tester |<------------+ +-------------| Tester |<------------+
| IPv6 address| [state table]| IPv4 address| | IPv6 address| [state table]| IPv4 address|
| +--------------------------------------+ | | +--------------------------------------+ |
| | | |
| +--------------------------------------+ | | +--------------------------------------+ |
| 2001:2::1 | DUT: | 198.19.0.1 | | 2001:2::1 | DUT: | 198.19.0.1 |
+------------>| Stateful NAT64 gateway |-------------+ +------------>| Stateful NAT64 gateway |-------------+
IPv6 address| [connection tracking table] | IPv4 address IPv6 address| [connection tracking table] | IPv4 address
+--------------------------------------+ +--------------------------------------+
]]></artwork> ]]></artwork>
<postamble></postamble>
</figure> </figure>
<t keepWithPrevious="true"/>
<t>As for the transport layer protocol, <xref target="RFC2544"
format="default"/> recommended testing with UDP, and it was also kept
in <xref target="RFC8219" format="default"/>. UDP is also kept for a
general recommendation; thus, the port numbers in the following text
are to be understood as UDP port numbers. The rationale and
limitations of this approach are discussed in <xref
target="udp_or_tcp" format="default"/>.</t>
<t>As for transport layer protocol, <xref target="RFC2544"/> recommen <t>The most important elements of the proposed benchmarking system are
ded defined as follows:</t>
testing with UDP, and it was kept also in <xref target="RFC8219"/
>. For
the general recommendation, UDP is also kept, thus the port numbe
rs in the
following text are to be understood as UDP port numbers. The rati
onale and limitations
of this approach are discussed in <xref target="udp_or_tcp"/>.</t
>
<t>The most important elements of the proposed benchmarking system ar <!-- [rfced] In the sentence below, may we clarify "also in the case of
e defined as follows. UDP using the same kind of entries as in the case of TCP" as follows?
<list style="symbols">
<t>Connection: Although UDP itself is a connection-less protocol,
stateful NATxy
gateways keep track of their translation mappings in the form of
a "connection"
also in the case of UDP using the same kind of entries as in the
case of TCP.</t>
<t>Connection tracking table: The stateful NATxy gateway uses a conne
ction
tracking table to be able to perform the stateful translation in
the server to
client direction. Its size, policy, and content are unknown to th
e Tester.</t>
<t>Four tuple: The four numbers that identify a connection are so
urce IP address,
source port number, destination IP address, destination port numb
er.</t>
<t>State table: The Responder of the Tester extracts the four tup
le from each received
test frame and stores it in its state table. Recommendation is gi
ven for writing and
reading order of the state table in <xref target="st_wr_order"/>.
</t>
<t>Initiator: The port of the Tester that may initiate a connecti
on through the
stateful DUT in the client-to-server direction. Theoretically, it
can use
any source and destination port numbers from the ranges recommend
ed by
<xref target="RFC4814"/>: if the used four tuple does not belong
to an existing
connection, the DUT will register a new connection into its conne
ction tracking table.</t>
<t>Responder: The port of the Tester that may not initiate a conn
ection through
the stateful DUT in the server-to-client direction. It may send o
nly frames that
belong to an existing connection. To that end, it uses four tuple
s that have been
previously extracted from the received test frames and stored in
its state table.</t>
<t>Test phase 1: Test frames are sent only by the Initiator to th
e
Responder through the DUT to fill both the connection tracking ta
ble of the DUT
and the state table of the Responder. This is a newly introduced
operation phase
for stateful NATxy benchmarking. The necessity of this test phase
is explained in
<xref target="prelim"/>.</t>
<t>Test phase 2: The measurement procedures defined by <xref targ
et="RFC8219"/>
(e.g. throughput, latency, etc.) are performed in this test phase
after the completion
of test phase 1. Test frames are sent as required (e.g. bidirecti
onal
test or unidirectional test in any of the two directions).</t>
</list>
</t>
<t>
One further definition is used in the text of this document:
<list style="symbols">
<t> Black box testing: It is a testing approach when the Tester i
s not aware of the
details of the internal structure and operation of the DUT. It ca
n send input to the
DUT and observe the output of the DUT.</t>
</list>
</t>
</section>
<section anchor="setup_term_multiple" title="When Testing with Multiple IP Addresses"> Original:
<t>The number of the necessary and available IP addresses are considere * Connection: Although UDP itself is a connection-less protocol,
d.</t> stateful NATxy gateways keep track of their translation mappings
in the form of a "connection" also in the case of UDP using the
same kind of entries as in the case of TCP.
<t>In <xref target="test_setup_sfnat44"/>, the single 198.19.0.1 IPv4 a Perhaps:
ddress is used
on the WAN side port of the stateful NAT44 gateway. However, in practic
e, not a single
IP address, but an IP address range is assigned to the WAN side port of
the stateful
NAT44 gateways. Its required size depends on the number of client nodes
and on the type
of the stateful NAT44 algorithm. (The traditional algorithm always repl
aces
the source port number, when a new connection is established. Thus it r
equires a larger
range than the extended algorithm, which replaces the source port numbe
r only when it
is necessary. Please refer to Table 1 and Table 2 of <xref target="LEN2
015"/>.)</t>
<t>When router testing is done, section 12 of <xref target="RFC2544"/> Connection: Although UDP itself is a connectionless protocol,
requires stateful NATxy gateways keep track of their translation mappings
testing first using a single source and destination IP address pair, an in the form of a "connection" as well as in the case of UDP using the
d then same kind of entries as in TCP.
using destination IP addresses from 256 different networks. The 16-23 b
its of
the 198.18.0.0/24 and 198.19.0.0/24 addresses can be used to express th
e 256 networks.
As this document does not deal with router testing, no multiple destina
tion networks are needed,
therefore, these bits are available for expressing multiple IP addresse
s that belong
to the same "/16" network. Moreover, both the 198.18.0.0/16 and the 198
.19.0.0/16
networks can be used on the right side of the test setup as private IP
addresses
from the 10.0.0.0/16 network are used on its left side.</t>
<figure anchor="test_setup_sfnat44_multi" align="center" title="Test setu -->
p for benchmarking
stateful NAT44 gateways using multiple IPv4 addresses">
<preamble></preamble>
<artwork align="left"><![CDATA[ <dl newline="false" spacing="normal">
10.0.0.2/16 – 10.0.255.254/16 198.19.0.0/15 - 198.19.255.254/15 <dt>Connection:</dt>
<dd>Although UDP itself is a connectionless protocol, stateful
NATxy gateways keep track of their translation mappings in the form
of a "connection" also in the case of UDP using the same kind of
entries as in the case of TCP.</dd>
<dt>Connection tracking table:</dt>
<dd>The stateful NATxy gateway uses a connection tracking table to
be able to perform the stateful translation in the server-to-client
direction. Its size, policy, and content are unknown to the
Tester.</dd>
<dt>Four tuple:</dt>
<dd>The four numbers that identify a connection are source IP
address, source port number, destination IP address, and destination
port number.</dd>
<dt>State table:</dt>
<dd>The Responder of the Tester extracts the four tuple from each
received test frame and stores it in its state table. A recommendation
is given for the writing and reading order of the state table in <xref
target="st_wr_order" format="default"/>.</dd>
<dt>Initiator:</dt>
<dd>The port of the Tester that may initiate a connection through
the stateful DUT in the client-to-server direction. Theoretically,
it can use any source and destination port numbers from the ranges
recommended by <xref target="RFC4814" format="default"/>: if the
used four tuple does not belong to an existing connection, the DUT
will register a new connection into its connection tracking
table.</dd>
<dt>Responder:</dt>
<dd>The port of the Tester that may not initiate a connection
through the stateful DUT in the server-to-client direction. It may
only send frames that belong to an existing connection. To that end,
it uses four tuples that have been previously extracted from the
received test frames and stores in its state table.</dd>
<dt>Test phase 1:</dt>
<dd>The test frames are sent only by the Initiator to the Responder
through the DUT to fill both the connection tracking table of the
DUT and the state table of the Responder. This is a newly introduced
operation phase for stateful NATxy benchmarking. The necessity of
this test phase is explained in <xref target="prelim"
format="default"/>.</dd>
<dt>Test phase 2:</dt>
<dd>The measurement procedures defined by <xref target="RFC8219"
format="default"/> (e.g., throughput, latency, etc.) are performed in
this test phase after the completion of test phase 1. Test frames
are sent as required (e.g., a bidirectional test or a unidirectional te
st
in any of the two directions).</dd>
</dl>
<t>One further definition is used in the text of this document:</t>
<dl newline="false" spacing="normal">
<dt>Black box testing:</dt>
<dd>A testing approach when the Tester is not aware of the
details of the internal structure and operation of the DUT. It can
send input to the DUT and observe the output of the DUT.</dd>
</dl>
</section>
<section anchor="setup_term_multiple" numbered="true" toc="default">
<name>When Testing with Multiple IP Addresses</name>
<t>This section considers the number of the necessary and available IP
addresses.</t>
<t>In <xref target="test_setup_sfnat44" format="default"/>, the single
198.19.0.1 IPv4 address is used on the WAN side port of the stateful
NAT44 gateway. However, in practice, it is not a single IP address,
but rather an IP address range that is assigned to the WAN side port
of the stateful NAT44 gateways. Its required size depends on the
number of client nodes and on the type of the stateful NAT44
algorithm. (The traditional algorithm always replaces the source port
number when a new connection is established. Thus, it requires a
larger range than the extended algorithm, which replaces the source
port number only when it is necessary. Please refer to Tables 1 and
2 of <xref target="LEN2015" format="default"/>.)</t>
<t>When router testing is done, <xref target="RFC2544"
sectionFormat="of" section="12"/> requires testing using a
single source and destination IP address pair first and then using
destination IP addresses from 256 different networks. The 16-23 bits
of the 198.18.0.0/24 and 198.19.0.0/24 addresses can be used to
express the 256 networks. As this document does not deal with router
testing, no multiple destination networks are needed; therefore, these
bits are available for expressing multiple IP addresses that belong to
the same "/16" network. Moreover, both the 198.18.0.0/16 and the
198.19.0.0/16 networks can be used on the right side of the test setup,
as private IP addresses from the 10.0.0.0/16 network are used on its
left side.</t>
<t keepWithNext="true"/>
<figure anchor="test_setup_sfnat44_multi">
<name>Test Setup for Benchmarking Stateful NAT44 Gateways Using Multip
le IPv4 Addresses</name>
<artwork align="left" name="" type="" alt=""><![CDATA[
10.0.0.2/16 - 10.0.255.254/16 198.19.0.0/15 - 198.19.255.254/15
\ +--------------------------------------+ / \ +--------------------------------------+ /
\ |Initiator Responder| / \ |Initiator Responder| /
+-------------| Tester |<------------+ +-------------| Tester |<------------+
| private IPv4| [state table]| public IPv4 | | private IPv4| [state table]| public IPv4 |
| +--------------------------------------+ | | +--------------------------------------+ |
| | | |
| +--------------------------------------+ | | +--------------------------------------+ |
| 10.0.0.1/16 | DUT: | public IPv4 | | 10.0.0.1/16 | DUT: | public IPv4 |
+------------>| Stateful NAT44 gateway |-------------+ +------------>| Stateful NAT44 gateway |-------------+
private IPv4| [connection tracking table] | \ private IPv4| [connection tracking table] | \
+--------------------------------------+ \ +--------------------------------------+ \
198.18.0.1/15 - 198.18.255.255/15 198.18.0.1/15 - 198.18.255.255/15
]]></artwork> ]]></artwork>
<postamble></postamble>
</figure> </figure>
<t>A possible solution for assigning multiple IPv4 addresses is shown <t keepWithPrevious="true"/>
in <t>A possible solution for assigning multiple IPv4 addresses is shown
<xref target="test_setup_sfnat44_multi"/>. On the left side, the in <xref target="test_setup_sfnat44_multi" format="default"/>. On the
private IP left side, the private IP address range is abundantly large. (The
address range is abundantly large. (The 16-31 bits were used for 16-31 bits were used for generating nearly 64k potential different
generating nearly 64k source addresses, but the 8-15 bits are also available if needed.) On
potential different source addresses, but the 8-15 bits are also the right side, the 198.18.0.0./15 network is used, and it was cut
available into two equal parts. (Asymmetric division is also possible, if
if needed.) On the right side, the 198.18.0.0./15 network is used needed.)</t>
, and it was <t>It should be noted that these are the potential address ranges. The
cut into two equal parts. (Asymmetric division is also possible, actual address ranges to be used are discussed in <xref
if needed.)</t> target="restr_port_range" format="default"/>.</t>
<t>In the case of stateful NAT64, a single "/64" IPv6 prefix contains
<t>It should be noted that these are the potential address ranges. Th a high number of bits to express different IPv6 addresses. <xref
e actual target="test_setup_sfnat64_multi" format="default"/> shows an example
address ranges to be used are discussed in <xref target="restr_po where bits 96-111 are used for that purpose.
rt_range"/>.</t> </t>
<t keepWithNext="true"/>
<t>In the case of stateful NAT64, a single "/64" IPv6 prefix contains <figure anchor="test_setup_sfnat64_multi">
a high <name>Test Setup for Benchmarking Stateful NAT64 Gateways Using
number of bits to express different IPv6 addresses. <xref target= Multiple IPv6 and IPv4 Addresses</name>
"test_setup_sfnat64_multi"/> <artwork align="left" name="" type="" alt=""><![CDATA[
shows an example, where bits 96-111 are used for that purpose.
</t>
<figure anchor="test_setup_sfnat64_multi" align="center" title="Test Setu
p for benchmarking
stateful NAT64 gateways using multiple IPv6 and IPv4 addresses">
<preamble></preamble>
<artwork align="left"><![CDATA[
2001:2::[0000-ffff]:0002/64 198.19.0.0/15 - 198.19.255.254/15 2001:2::[0000-ffff]:0002/64 198.19.0.0/15 - 198.19.255.254/15
\ +--------------------------------------+ / \ +--------------------------------------+ /
IPv6 \ |Initiator Responder| / IPv6 \ |Initiator Responder| /
+-------------| Tester |<------------+ +-------------| Tester |<------------+
| addresses | [state table]| public IPv4 | | addresses | [state table]| public IPv4 |
| +--------------------------------------+ | | +--------------------------------------+ |
| | | |
| +--------------------------------------+ | | +--------------------------------------+ |
| 2001:2::1/64| DUT: | public IPv4 | | 2001:2::1/64| DUT: | public IPv4 |
+------------>| Stateful NAT64 gateway |-------------+ +------------>| Stateful NAT64 gateway |-------------+
IPv6 address | [connection tracking table] | \ IPv6 address | [connection tracking table] | \
+--------------------------------------+ \ +--------------------------------------+ \
198.18.0.1/15 - 198.18.255.255/15 198.18.0.1/15 - 198.18.255.255/15
]]></artwork> ]]></artwork>
<postamble></postamble>
</figure> </figure>
<t keepWithPrevious="true"/>
</section>
</section>
<section anchor="method" numbered="true" toc="default">
<name>Recommended Benchmarking Method</name>
<section anchor="restr_port_range" numbered="true" toc="default">
<name>Restricted Number of Network Flows</name>
<t>When a single IP address pair is used for testing, then the number
of network flows is determined by the number of source and
destination port number combinations. </t>
<!-- [rfced] For clarity, may we update "in the order of a few times ten
thousand" to "in the order of a few tens of thousands"?
Original:
If it is possible, the size of the source port number range SHOULD be
larger (e.g. in the order of a few times ten thousand), whereas the size of
the destination port number range SHOULD be smaller (may vary from a few to
several hundreds or thousands as needed).
Perhaps:
If it is possible, the size of the source port number range SHOULD be
larger (e.g., in the order of a few tens of thousands), whereas the size of
the destination port number range SHOULD be smaller (may vary from a few to
several hundreds or thousands as needed).
-->
<t>The Initiator <bcp14>SHOULD</bcp14> use restricted ranges for
source and destination port numbers to avoid the exhaustion of the
connection tracking table capacity of the DUT as described in <xref
target="problem" format="default"/>. If it is possible, the size of
the source port number range <bcp14>SHOULD</bcp14> be larger (e.g., in
the order of a few times ten thousand), whereas the size of the
destination port number range <bcp14>SHOULD</bcp14> be smaller (e.g., it
may
vary from a few to several hundreds or thousands as needed). The
rationale is that source and destination port numbers that can be
observed in Internet traffic are not symmetrical. Whereas source
port numbers may be random, there are a few very popular destination
port numbers (e.g., 443 or 80; see <xref target="IIR2020"
format="default"/>), and others hardly occur. Additionally, it was found
that
their role is also asymmetric in the Linux kernel routing hash
function <xref target="LEN2020" format="default"/>.</t>
<t>However, in some special cases, the size of the source port range
is limited. For example, when benchmarking the Customer Edge (CE) and
Border Relay (BR) of a Mapping of Address and Port using Translation
(MAP-T) system <xref target="RFC7599" format="default"/> together (as
a compound system performing stateful NAT44), the source port
range is limited to the number of source port numbers assigned to each
subscriber. (It could be as low as 2048 ports.)</t>
<!-- [rfced] FYI - To improve readability, we have reformatted the text below
to read as a bulleted list. Please let us know any objections.
Original:
When multiple IP addresses are used, then the port number ranges
should be even more restricted, as the number of potential network
flows is the product of the size of the source IP address range, the
size of the source port number range, the size of the destination IP
address range, and the size of the destination port number range.
Current:
When multiple IP addresses are used, then the port number ranges
should be even more restricted, as the number of potential network
flows is the product of the size of:
* the source IP address range,
* the source port number range,
* the destination IP address range, and
* the destination port number range.
-->
<t>When multiple IP addresses are used, then the port number ranges
should be even more restricted, as the number of potential network
flows is the product of the size of:</t>
<ul>
<li>the source IP address range,</li>
<li>the source port number range,</li>
<li>the destination IP address range, and</li>
<li>the destination port number range.</li>
</ul>
<t>In addition, the recommended method requires the enumeration of all
their possible combinations in test phase 1 as described in <xref
target="ctrl_conntrack" format="default"/>.</t>
<t>The number of network flows can be used as a parameter. The
performance of the stateful NATxy gateway <bcp14>MAY</bcp14> be
examined as a function of this parameter as described in <xref
target="sc_net_flows" format="default"/>.</t>
</section> </section>
</section> <section anchor="prelim" numbered="true" toc="default">
<name>Test Phase 1</name>
<t>Test phase 1 serves two purposes:</t>
<section anchor="method" title="Recommended Benchmarking Method"> <!-- [rfced] How may we clarify "that is throughput" in the text below?
<section anchor="restr_port_range" title="Restricted Number of Network Flows"> Original:
<t>When a single IP address pair is used for testing then the number of Test phase 1 serves two purposes:
network
flows is determined by the number of source port number destination por
t number
combinations. </t>
<t>The Initiator SHOULD use restricted ranges for source and destinatio 1. The connection tracking table of the DUT is filled. It is
n important, because its maximum connection establishment rate may
port numbers to avoid the exhaustion of the connection tracking table be lower than its maximum frame forwarding rate (that is
capacity of the DUT as described in <xref target="problem"/>. throughput).
If it is possible, the size of the source port number range SHOULD be l
arger (e.g. in the order
of a few times ten thousand), whereas the size of the destination port
number
range SHOULD be smaller (may vary from a few to several hundreds or tho
usands
as needed).
The rationale is that source and destination port numbers that can be o
bserved in
the Internet traffic are not symmetrical. Whereas source port numbers m
ay be random,
there are a few very popular destination port numbers (e.g. 443, 80, et
c.,
see <xref target="IIR2020"/>), and others hardly occur. And it was
found that their role is also asymmetric in the Linux kernel routing ha
sh
function <xref target="LEN2020"/>.</t>
<t>However, in some special cases, the size of the source port range is
limited. E.g.
when benchmarking the CE and BR of a MAP-T <xref target="RFC7599"/> sys
tem together (as a compound system
performing stateful NAT44), then the source port range is limited to th
e number of
source port numbers assigned to each subscriber. (It could be as low as
2048 ports.) </t>
<t>When multiple IP addresses are used, then the port number ranges sho Perhaps:
uld be even
more restricted, as the number of potential network flows is the produc
t of the size
of the source IP address range, the size of the source port number rang
e, the size of the
destination IP address range, and the size of the destination port numb
er range.
And the recommended method requires the enumeration of all their possib
le combinations in test
phase 1 as described in <xref target="ctrl_conntrack"/>.</t>
<t>The number of network flows can be used as a parameter. The performa Test phase 1 serves two purposes:
nce of the
stateful NATxy gateway MAY be examined as a function of this parameter
as described
in <xref target="sc_net_flows"/>.</t>
</section>
<section anchor="prelim" title="Test Phase 1"> 1. The connection tracking table of the DUT is filled. This is important
<t>Test phase 1 serves two purposes: because its maximum connection establishment rate may be lower than its
<list style="numbers"> maximum frame forwarding rate (that is, its throughput).
<t>The connection tracking table of the DUT is filled. It is im
portant,
because its maximum connection establishment rate may be lower
than its maximum
frame forwarding rate (that is throughput).</t>
<t>The state table of the Responder is filled with valid four t
uples. It is
a precondition for the Responder to be able to transmit frames
that belong to
connections that exist in the connection tracking table of the
DUT.</t>
</list>
Whereas the above two things are always necessary before test pha
se 2,
test phase 1 can be used without test phase 2. It is done so
when the maximum connection establishment rate is measured (as de
scribed in
<xref target="meas_max_conn_est_rate"/>).
</t>
<t>Test phase 1 MUST be performed before all tests performed in
test phase 2. The following things happen in test phase 1:
<list style="numbers">
<t>The Initiator sends test frames to the Responder through the
DUT at a
specific frame rate.</t>
<t>The DUT performs the stateful translation of the test frames
and it also
stores the new connections in its connection tracking table.</t
>
<t>The Responder receives the translated test frames and update
s its state
table with the received four tuples. The responder transmits no
test frames
during test phase 1.</t>
</list>
</t>
<t>When test phase 1 is performed in preparation for
test phase 2, the applied frame rate SHOULD be safely lower than
the maximum connection establishment rate. (It implies that maxim
um connection
establishment rate measurement MUST be performed first.)
Please refer to <xref target="ctrl_conntrack"/> for further condi
tions regarding
timeout and the enumeration of all possible four tuples.</t>
</section>
<section anchor="consider_stateful" title="Consideration of the Cases o -->
f Stateful Operation">
<t>The authors consider the most important events that may happen
during the operation
of a stateful NATxy gateway, and the Actions of the gateway as fo
llows.
<list style="numbers">
<t>EVENT: A packet not belonging to an existing connection arri
ves in the client-to-server
direction. ACTION: A new connection is registered into the conn
ection tracking
table and the packet is translated and forwarded.</t>
<t>EVENT: A packet not belonging to an existing connection arri
ves in the server-to-client
direction. ACTION: The packet is discarded.</t>
<t>EVENT: A packet belonging to an existing connection arrives
(in any direction).
ACTION: The packet is translated and forwarded and the timeout
counter of the corresponding
connection tracking table entry is reset.</t>
<t>EVENT: A connection tracking table entry times out. ACTION:
The entry is deleted from
the connection tracking table.</t>
</list>
</t>
<t>Due to "black box" testing, the Tester is not able to directly
examine (or delete) the entries
of the connection tracking table. But the entries can be and MUST
be controlled by setting
an appropriate timeout value and carefully selecting the port num
bers of the packets
(as described in <xref target="ctrl_conntrack"/>) to be able to p
roduce meaningful and
repeatable measurement results.
</t>
<t>This document aims to support the measurement of the following
performance characteristics
of a stateful NATxy gateway:
<list style="numbers">
<t>maximum connection establishment rate</t>
<t>all "classic" performance metrics like throughput, frame los
s rate, latency, etc.</t>
<t>connection tear-down rate</t>
<t>connection tracking table capacity</t>
</list>
</t>
</section>
<section anchor="ctrl_conntrack" title="Control of the Connection Track <ol spacing="normal" type="1">
ing Table Entries"> <li>
<t>It is necessary to control the connection tracking table entri <t>The connection tracking table of the DUT is filled. This is
es important because its maximum connection establishment rate may
of the DUT to achieve clear conditions for the measurements. One be lower than its maximum frame forwarding rate (that is
can simply throughput).</t>
achieve the following two extreme situations: </li>
<list style="numbers"> <li>
<t>All frames create a new entry in the connection tracking tab <t>The state table of the Responder is filled with valid four
le of the DUT and no tuples. It is a precondition for the Responder to be able to
old entries are deleted during the test. This is required for m transmit frames that belong to connections that exist in the
easuring the maximum connection tracking table of the DUT.</t>
connection establishment rate.</t> </li>
<t>No new entries are created in the connection tracking table </ol>
of the DUT and no old
ones are deleted during the test. This is ideal for the measure
ments to be executed in phase 2,
like throughput, latency, etc.</t>
</list>
</t>
<t>From this point, the following two assumptions are used: <t>Whereas the above two things are always necessary before test phase
<list style="numbers"> 2, test phase 1 can be used without test phase 2. This is done when
<t>The connection tracking table of the stateful NATxy is large the maximum connection establishment rate is measured (as described in
enough to store all <xref target="meas_max_conn_est_rate" format="default"/>).</t>
connections defined by the different four tuples.</t>
<t>Each experiment is started with an empty connection tracking
table. (It can be ensured
by deleting its content before the experiment.)</t>
</list>
</t>
<t>The first extreme situation can be achieved by <t>Test phase 1 <bcp14>MUST</bcp14> be performed before all tests are
<list style="symbols"> performed in test phase 2. The following things happen in test phase
<t>using different four tuples for every single test frame in t 1:</t>
est phase 1 and</t>
<t> setting the UDP timeout of the NATxy gateway to a value hig
her than the length of
test phase 1.</t>
</list>
</t>
<t>The second extreme situation can be achieved by <ol spacing="normal" type="1">
<list style="symbols"> <li>
<t>enumerating all possible four tuples in test phase 1 and</t> <t>The Initiator sends test frames to the Responder through the
<t>setting the UDP timeout of the NATxy gateway to a value high DUT at a specific frame rate.</t>
er than the length of </li>
test phase 1 plus the gap between the two phases plus the lengt <li>
h of test phase 2.</t> <t>The DUT performs the stateful translation of the test frames,
</list> and it also stores the new connections in its connection tracking
</t> table.</t>
</li>
<li>
<t>The Responder receives the translated test frames and updates
its state table with the received four tuples. The Responder
transmits no test frames during test phase 1.</t>
</li>
</ol>
<t>When test phase 1 is performed in preparation for test phase 2, the
applied frame rate <bcp14>SHOULD</bcp14> be safely lower than the
maximum connection establishment rate. (It implies that maximum
connection establishment rate measurement <bcp14>MUST</bcp14> be
performed first.) Please refer to <xref target="ctrl_conntrack"
format="default"/> for further conditions regarding timeout and the
enumeration of all possible four tuples.</t>
</section>
<t> <section anchor="consider_stateful" numbered="true" toc="default">
<xref target="RFC4814"/> REQUIRES pseudorandom port numbers, whic <name>Consideration of the Cases of Stateful Operation</name>
h the authors believe is a good <t>The authors consider the most important events that may happen
approximation of the distribution of the source port numbers a NA during the operation of a stateful NATxy gateway and the Actions of
Txy gateway on the the gateway as follows.</t>
Internet may face with.
</t>
<t> <ol>
It should be noted that although the enumeration of all possible <li>
four tuples <t>EVENT: A packet not belonging to an existing connection arrives
is not a requirement for the first extreme situation and the usag in the client-to-server direction.</t>
e of <t>ACTION: A new connection is registered into the connection
different four tuples in test phase 1 is not a tracking table, and the packet is translated and forwarded.</t>
requirement for the second extreme situation, pseudorandom </li>
enumeration of all possible four tuples in test phase 1 <li>
is a good solution in both cases. It may be computing efficiently <t>EVENT: A packet not belonging to an existing connection
generated by preparing a random permutation of the previously arrives in the server-to-client direction.</t>
enumerated all possible four tuples using Dustenfeld's random shu <t>ACTION: The packet is discarded.</t>
ffle </li>
algorithm <xref target="DUST1964"/>. <li>
</t> <t>EVENT: A packet belonging to an existing connection arrives
(in any direction).</t>
<t>ACTION: The packet is translated and forwarded, and the
timeout counter of the corresponding connection tracking table
entry is reset.</t>
</li>
<li>
<t>EVENT: A connection tracking table entry times out.</t>
<t>ACTION: The entry is deleted from the connection tracking
table.</t>
</li>
</ol>
<t>The enumeration of the four tuples in increasing or decreasing <t>Due to "black box" testing, the Tester is not able to directly
order examine (or delete) the entries of the connection tracking
(or in any other specific order) MAY be used as an additional mea table. However, the entries can and <bcp14>MUST</bcp14> be
surement. controlled by setting an appropriate timeout value and carefully
</t> selecting the port numbers of the packets (as described in <xref
</section> target="ctrl_conntrack" format="default"/>) to be able to produce
meaningful and repeatable measurement results.</t>
<t>This document aims to support the measurement of the following
performance characteristics of a stateful NATxy gateway:</t>
<ul spacing="normal">
<li>
<t>maximum connection establishment rate</t>
</li>
<li>
<t>all "classic" performance metrics like throughput, frame loss rat
e, latency, etc.</t>
</li>
<li>
<t>connection tear-down rate</t>
</li>
<li>
<t>connection tracking table capacity</t>
</li>
</ul>
</section>
<section anchor="meas_max_conn_est_rate" title="Measurement of the Maxi <section anchor="ctrl_conntrack" numbered="true" toc="default">
mum Connection Establishment Rate"> <name>Control of the Connection Tracking Table Entries</name>
<t>The maximum connection establishment rate is an important characte <t>It is necessary to control the connection tracking table entries of
ristic of the DUT to achieve clear conditions for the measurements. One can
the stateful NATxy gateway and its determination is necessary for simply achieve the following two extreme situations:</t>
the safe
execution of test phase 1 (without frame loss) before test phase
2.
</t>
<t>The measurement procedure of the maximum connection establishm
ent rate is
very similar to the throughput measurement procedure defined in
<xref target="RFC2544"/>.
</t>
<t>Procedure: The Initiator sends a specific number of test frame
s using all
different four tuples at a specific rate through the DUT.
The Responder counts the frames that are successfully translated
by the DUT.
If the count of offered frames is equal to the count of received
frames, the rate of the offered stream is raised and the test is
rerun.
If fewer frames are received than were transmitted, the rate of t
he offered
stream is reduced and the test is rerun.
</t>
<t>The maximum connection establishment rate is the fastest rate
at which
the count of test frames successfully translated by the DUT is eq
ual to the number
of test frames sent to it by the Initiator.
</t>
<t>Note: In practice, the usage of binary search is RECOMMENDED.<
/t>
</section>
<section anchor="validation_of_conn" title="Validation of Connection Es <ol spacing="normal">
tablishment"> <li>
<t>Due to "black box" testing, the entries of the connection tracking All frames create a new entry in the connection tracking table
table of of the DUT, and no old entries are deleted during the test. This is
the DUT may not be directly examined, but the presence of the con required for measuring the maximum connection establishment
nections can be rate.
checked easily by sending frames from the Responder to the Initia </li>
tor in <li>
test phase 2 using all four tuples stored in the state table of t No new entries are created in the connection tracking table of
he Tester the DUT, and no old ones are deleted during the test. This is ideal
(at a low enough frame rate). The arrival of all test frames indi for the measurements to be executed in phase 2, like throughput,
cates that the latency, etc.
connections are indeed present. </li>
</t> </ol>
<t>Procedure: When all the desired N number of test frames were s <t>From this point, the following two assumptions are used:</t>
ent by the Initiator
to the Receiver at frame rate R in test phase 1 for the maximum c
onnection
establishment rate measurement, and the Receiver has successfully
received all
the N frames, the establishment of the connections is checked in
test
phase 2 as follows:
<list style="symbols">
<t>The Responder sends test frames to the Initiator at frame ra
te r=R*alpha,
for the duration of N/r using a different four tuple from its s
tate table for
each test frame.</t>
<t>The Initiator counts the received frames, and if all N frame
s are arrived
then the R frame rate of the maximum connection establishment r
ate measurement
(performed in test phase 1) is raised for the next iteration,
otherwise lowered (as well as in the case if test frames were m
issing
in the preliminary test phase).</t>
</list>
</t>
<t>Notes:
<list style="symbols">
<t>The alpha is a kind of "safety factor", it aims to make su
re that
the frame rate used for the validation is not too high, a
nd test may fail only
in the case if at least one connection is not present in
the connection
tracking table of the DUT. (So alpha should be typically
less than 1, e.g.
0.8 or 0.5.)
</t>
<t>The duration of N/r and the frame rate of r means that
N frames are sent for validation.</t>
<t>The order of four tuple selection is arbitrary provide
d that all four tuples MUST be used.</t>
<t>Please refer to <xref target="meas_contr_capacity"/> f
or a short analysis
of the operation of the measurement and what problems may
occur.</t>
</list>
</t>
</section>
<section anchor="real_test" title="Test Phase 2"> <ol spacing="normal" type="1">
<t>As for the traffic direction, there are three possible cases durin <li anchor="assumption1">
g The connection tracking table of the stateful NATxy is large
test phase 2: enough to store all connections defined by the different four
<list style="symbols"> tuples.
<t>bidirectional traffic: The Initiator sends test frames to th </li>
e Responder and <li anchor="assumption2">
the Responder sends test frames to the Initiator.</t> Each experiment is started with an empty connection tracking
<t>unidirectional traffic from the Initiator to the Responder: table. (This can be ensured by deleting its content before the
The Initiator experiment.)
sends test frames to the Responder but the Responder does not s </li>
end test frames to </ol>
the Initiator.</t>
<t>unidirectional traffic from the Responder to the Initiator:
The Responder
sends test frames to the Initiator but the Initiator does not s
end test frames to
the Responder.</t>
</list>
</t>
<t>If the Initiator sends test frames, then it uses pseudorandom
source port numbers and
destination port numbers from the restricted port number ranges.
(If it uses multiple
source and/or destination IP addresses, then their ranges are als
o limited.)
The responder receives the test frames, updates its state table,
and processes the test
frames as required by the given measurement procedure (e.g. only
counts them for the
throughput test, handles timestamps for latency or PDV tests, etc
.).
</t>
<t>If the Responder sends test frames, then it uses the four tupl
es from its state
table. The reading order of the state table may follow different
policies (discussed
in <xref target="st_wr_order"/>). The Initiator receives the test
frames and
processes them as required by the given measurement procedure.
</t>
<t>
As for the actual measurement procedures, the usage of the update
d ones
from Section 7 of <xref target="RFC8219"/> is RECOMMENDED.
</t>
</section>
<section anchor="meas_conn_tear_down_rate" title="Measurement of the Co <t>The first extreme situation can be achieved by:</t>
nnection Tear-down Rate"> <ul spacing="normal">
<t>Connection tear-down can cause significant load for the NATxy <li>
gateway. <t>using different four tuples for every single test frame in test p
The connection tear-down performance can be measured as follows: hase 1 and</t>
<list style="numbers"> </li>
<t>Load a certain number of connections (N) into the connection <li>
tracking table of the DUT (in the same way as done to measure t <t>setting the UDP timeout of the NATxy gateway to a value higher
he than the length of test phase 1.</t>
maximum connection establishment rate).</t> </li>
<t>Record TimestampA.</t> </ul>
<t>Delete the content of the connection tracking table of the D <t>The second extreme situation can be achieved by:</t>
UT.</t>
<t>Record TimestampB.</t> <ul spacing="normal">
</list> <li>
The connection tear-down rate can be computed as: <t>enumerating all possible four tuples in test phase 1 and</t>
</t> </li>
<t>connection tear-down rate = N / ( TimestampB - TimestampA) <li>
<t>setting the UDP timeout of the NATxy gateway to a value higher
than the length of test phase 1 plus the gap between the two
phases plus the length of test phase 2.</t>
</li>
</ul>
<!--[rfced] As "REQUIRES" is not a key word per RFCs 2119/8174, may we
rephrase this sentence to use "REQUIRED"?
Original:
[RFC4814] REQUIRES pseudorandom port numbers, which the authors
believe is a good approximation of the distribution of the source
port numbers a NATxy gateway on the Internet may face with.
Perhaps:
As described in [RFC4814], pseudorandom port numbers are REQUIRED,
which the authors believe is a good approximation of the distribution
of the source port numbers a NATxy gateway on the Internet may face with.
-->
<t><xref target="RFC4814" format="default"/> REQUIRES pseudorandom
port numbers, which the authors believe is a good approximation of the
distribution of the source port numbers a NATxy gateway on the
Internet may be faced with.
</t> </t>
<t>The connection tear-down rate SHOULD be measured for various v
alues of N.
</t>
<t>It is assumed that the content of the connection tracking table may b
e deleted
by an out-of-band control mechanism specific to the given NATxy g
ateway implementation.
(E.g. by removing the appropriate kernel module under Linux.)
</t>
<t>It is noted that the performance of removing the entire content of th
e connection
tracking table at one time may be different from removing all the
entries one by one.
</t>
</section> <!-- [rfced] For clarity, how may we rephrase "it may be computing efficiently
generated by preparing" in the text below?
<section anchor="meas_contr_capacity" title="Measurement of the Connect Original:
ion Tracking Table Capacity">
<t>The connection tracking table capacity is an important metric
of stateful
NATxy gateways. Its measurement is not easy, because an elementar
y
step of a validated maximum connection establishment rate measure
ment (defined in
<xref target="validation_of_conn"/>) may have only a few distinct
observable outcomes,
but some of them may have different root causes:
<list style="numbers">
<t>During test phase 1, the number of test frames received by t
he
Responder is less than the number of test frames sent by the In
itiator.
It may have different root causes, including:
<list style="numbers">
<t>The R frame sending rate was higher than the maximum conne
ction
establishment rate. (Note that now the maximum connection
establishment rate is considered unknown because one can
not measure the
maximum connection establishment without assumption 1 in
<xref target="ctrl_conntrack"/>!)
This root cause may be eliminated by lowering the R rate
and re-executing
the test. (This step may be performed multiple times, whi
le R>0.)</t>
<t>The capacity of the connection tracking table of the D
UT has been
exhausted. (And either the DUT does not want to delete
connections
or the deletion of the connections makes it slower. Thi
s case is not
investigated further in test phase 1.)</t>
</list>
</t>
<t>During test phase 1, the number of test frames received by t
he
Responder equals the number of test frames sent by the Initiato
r.
In this case, the connections are validated in test phase 1.
The validation may have two kinds of observable results:
<list style="numbers">
<t>The number of validation frames received by the Initiator
equals the number of validation frames sent by the Respon
der.
(It proves that the capacity of the connection tracking t
able of
the DUT is enough and both R and r were chosen properly.)
</t>
<t>The number of validation frames received by the Initia
tor
is less than the number of validation frames sent by the
Responder.
This phenomenon may have various root causes:
<list style="numbers">
<t>The capacity of the connection tracking table of the
DUT has been
exhausted. (It does not matter, whether some existing c
onnections are
discarded and new ones are stored, or the new connectio
ns are discarded.
Some connections are lost anyway, and it makes validati
on fail.)</t>
<t>The R frame sending rate used by the Initiator was t
oo high in
test phase 1 and thus some connections were not establi
shed,
even though all test frames arrived at the Responder. T
his root cause
may be eliminated by lowering the R rate and re-executi
ng the test.
(This step may be performed multiple times, while R>0.)
</t>
<t>The r frame sending rate used by the Responder was t
oo high in
test phase 2 and thus some test frames did not arrive a
t the Initiator, even
though all connections were present in the connection t
racking table of the DUT.
This root cause may be eliminated by lowering the r rat
e and re-executing the test.
(This step may be performed multiple times, while r>0.)
</t>
</list>
And here is the problem: as the above three root causes a
re indistinguishable,
it is not easy to decide, whether R or r should be decrea
sed.
</t>
</list>
</t>
</list>
</t>
<t>Experience shows that the DUT may collapse if its memory is ex It may be computing efficiently generated by preparing a
hausted. random permutation of the previously enumerated all possible four
Such a situation may make the connection tuples using Durstenfeld's random shuffle algorithm [DUST1964].
tracking table capacity measurements rather inconvenient. This po
ssibility is included
in the recommended measurement procedure, but the detection and e
limination of such
a situation is not addressed. (E.g. how the algorithm can reset t
he DUT.)
</t>
<t>For the connection tracking table size measurement, first one Perhaps:
needs a safe
number: C0. It is a precondition, that C0 number of connections c
an surely be
stored in the connection tracking table of the DUT. Using C0, one
can determine
the maximum connection establishment rate using C0 number of conn
ections.
It is done with a binary search using validation. The result is R
0. The values
C0 and R0 will serve as "safe" starting values for the following
two searches.
</t>
<t>First, an exponential search is performed to find the order of Efficient computing may be generated by preparing a
magnitude of the random permutation of the previously enumerated all possible four
connection tracking table capacity. The search stops if the DUT c tuples using Durstenfeld's random shuffle algorithm [DUST1964].
ollapses OR
the maximum connection establishment rate severely drops (e.g. to
its one tenth)
due to doubling the number of connections.
</t>
<t>Then, the result of the exponential search gives the order of -->
magnitude of
the size of the connection tracking table. Before disclosing the
possible algorithms to
determine the exact size of the connection tracking table, three
possible
replacement policies for the NATxy gateway are considered:
<list style="numbers">
<t>The gateway does not delete any live connections until their
timeout expires.</t>
<t>The gateway replaces the live connections according to LRU (
least recently used) policy.</t>
<t>The gateway does a garbage collection when its connection tr
acking table is full
and a frame with a new four tuple arrives. During the garbage c
ollection, it deletes the K
least recently used connections, where K is greater than 1.</t>
</list>
Now, it is examined what happens and how many validation frames a
rrive in the there cases.
Let the size of the connection tracking table be S, and the numbe
r of preliminary
frames be N, where S is less than N.
<list style="numbers">
<t>The connections defined by the first S test frames are regis
tered into
the connection tracking table of the DUT, and the last N-S conn
ections are lost.
(It is another question if the last N-S test frames are transla
ted and
forwarded in test phase 1 or simply dropped.) During validation
, the validation
frames with four tuples corresponding to the first S test frame
s will arrive at the
Initiator and the other N-S validation frames will be lost.</t>
<t>All connections are registered into the connection tracking
table of the DUT,
but the first N-S connections are replaced (and thus lost). Dur
ing validation,
the validation frames with four tuples corresponding to the las
t S test frames
will arrive to the Initiator, and the other N-S validation fram
es will be lost. </t>
<t>Depending on the values of K, S, and N, maybe less than S co
nnections will survive.
In the worst case, only S-K+1 validation frames arrive, even th
ough, the size of
the connection tracking table is S.</t>
</list>
If one knows that the stateful NATxy gateway uses the first or se
cond replacement
policy and one also knows that both R and r rates are low enough,
then the final
step of determining the size of the connection tracking table is
simple. If the Responder
sent N validation frames and the Initiator received N' of them, t
hen the size of the
connection tracking table is N'.
</t>
<t>In the general case, a binary search is performed to find the <t>Although the enumeration of all possible four tuples is not a
exact value of the connection requirement for the first extreme situation and the usage of
tracking table capacity within E error. The search chooses the lo different four tuples in test phase 1 is not a requirement for the
wer half of second extreme situation, pseudorandom
the interval if the DUT collapses OR the maximum connection estab enumeration of all possible four tuples in test phase 1 is a good
lishment solution in both cases. It may be computing efficiently generated by
rate severely drops (e.g. to its half) otherwise it chooses the h preparing a random permutation of the previously enumerated all
igher half. possible four tuples using Durstenfeld's random shuffle algorithm <xref
The search stops if the size of the interval is less than the E e target="DUST1964" format="default"/>.</t>
rror.
</t>
<t>The algorithms for the general case are defined using C like p <t>The enumeration of the four tuples in increasing or decreasing
seudocode in order (or in any other specific order) <bcp14>MAY</bcp14> be used as
<xref target="meas_contr_capacity_algo"/>. In practice, this algo an additional measurement.</t>
rithm may
be made more efficient in a way that the binary search for the ma
ximum
connection establishment rate stops, if an elementary test fails
at a rate
under RS*beta or RS*gamma during the external search or during th
e final
binary search for the capacity of the connection tracking table,
respectively.
(This saves a high amount of execution time by eliminating the lo
ng-lasting tests at
low rates.)
</t>
<figure anchor="meas_contr_capacity_algo" align="center" title="Measurem </section>
ent of the Connection Tracking Table Capacity">
<sourcecode type="pseudocode"><![CDATA[ <section anchor="meas_max_conn_est_rate" numbered="true" toc="default">
<name>Measurement of the Maximum Connection Establishment Rate</name>
<t>The maximum connection establishment rate is an important
characteristic of the stateful NATxy gateway, and its determination is
necessary for the safe execution of test phase 1 (without frame loss)
before test phase 2.
</t>
<t>The measurement procedure of the maximum connection establishment
rate is very similar to the throughput measurement procedure defined
in <xref target="RFC2544" format="default"/>.
</t>
<!-- [rfced] FYI - We have reformatted the text below to read as a bulleted
list to improve readability. Please review and let us know of any objections.
Original:
Procedure: The Initiator sends a specific number of test frames using
all different four tuples at a specific rate through the DUT. The
Responder counts the frames that are successfully translated by the
DUT. If the count of offered frames is equal to the count of
received frames, the rate of the offered stream is raised and the
test is rerun. If fewer frames are received than were transmitted,
the rate of the offered stream is reduced and the test is rerun.
Current:
The procedure is as follows:
* The Initiator sends a specific number of test frames using all
different four tuples at a specific rate through the DUT.
* The Responder counts the frames that are successfully translated
by the DUT.
* If the count of offered frames is equal to the count of received
frames, the rate of the offered stream is raised and the test is
rerun.
* If fewer frames are received than were transmitted, the rate of
the offered stream is reduced and the test is rerun.
-->
<t>The procedure is as follows:</t>
<ul>
<li>The Initiator sends a specific number of test frames using all
different four tuples at a specific rate through the DUT.</li>
<li>The Responder counts the frames that are successfully translated
by the DUT.</li>
<li>If the count of offered frames is equal to the count of received
frames, the rate of the offered stream is raised and the test is
rerun.</li>
<li>If fewer frames are received than were transmitted, the rate of
the offered stream is reduced and the test is rerun.</li>
</ul>
<t>The maximum connection establishment rate is the fastest rate at
which the count of test frames successfully translated by the DUT is
equal to the number of test frames sent to it by the Initiator.
</t>
<!-- [rfced] Please review whether any of the notes in this document
should be in the <aside> element. It is defined as "a container for
content that is semantically less important or tangential to the
content that surrounds it"
(https://authors.ietf.org/en/rfcxml-vocabulary#aside).
-->
<t>Note: In practice, the usage of binary search is
<bcp14>RECOMMENDED</bcp14>.</t>
</section>
<section anchor="validation_of_conn" numbered="true" toc="default">
<name>Validation of Connection Establishment</name>
<t>Due to "black box" testing, the entries of the connection tracking
table of the DUT may not be directly examined. However, the presence of
the
connections can be checked easily by sending frames from the Responder
to the Initiator in test phase 2 using all four tuples stored in the
state table of the Tester (at a low enough frame rate). The arrival of
all test frames indicates that the connections are indeed present.
</t>
<t>The procedure is as follows:</t>
<t>When all the desired N number of test frames are sent by the
Initiator to the Receiver at frame rate R in test phase 1 for the
maximum connection establishment rate measurement and the Receiver
has successfully received all the N frames, the establishment
of the connections is checked in test phase 2 as follows:</t>
<ul>
<li>
The Responder sends test frames to the Initiator at frame rate
r=R*alpha for the duration of N/r, using a different four tuple
from its state table for each test frame.
</li>
<li>
The Initiator counts the received frames, and if all N frames
have arrived, then the R frame rate of the maximum connection
establishment rate measurement (performed in test phase 1) is
raised for the next iteration; otherwise, it is lowered (as well a
s in
the case that test frames were missing in the preliminary test
phase, as well).
</li>
</ul>
<t>Notes:</t>
<ul spacing="normal">
<li>
The alpha is a kind of "safety factor"; it aims to make sure
that the frame rate used for the validation is not too high, and t
he
test may fail only in the case of if at least one connection is no
t
present in the connection tracking table of the DUT. (Therefore, a
lpha
should be typically less than 1, e.g., 0.8 or 0.5.)
</li>
<li>
The duration of N/r and the frame rate of r means that N frames
are sent for validation.
</li>
<li>
The order of four tuple selection is arbitrary, provided that
all four tuples <bcp14>MUST</bcp14> be used.
</li>
<li>
Please refer to <xref target="meas_contr_capacity"
format="default"/> for a short analysis of the operation of the
measurement and what problems may occur.
</li>
</ul>
</section>
<section anchor="real_test" numbered="true" toc="default">
<name>Test Phase 2</name>
<t>As for the traffic direction, there are three possible cases
during test phase 2:</t>
<ol spacing="normal" type="1">
<li>
<t>Bidirectional traffic: The Initiator sends test frames to the
Responder, and the Responder sends test frames to the
Initiator.</t>
</li>
<li>
<t>Unidirectional traffic from the Initiator to the Responder: The
Initiator sends test frames to the Responder, but the Responder
does not send test frames to the Initiator.</t>
</li>
<li>
<t>Unidirectional traffic from the Responder to the Initiator: The
Responder sends test frames to the Initiator, but the Initiator
does not send test frames to the Responder.</t>
</li>
</ol>
<t>If the Initiator sends test frames, then it uses pseudorandom
source port numbers and destination port numbers from the restricted
port number ranges. (If it uses multiple source and/or destination IP
addresses, then their ranges are also limited.) The Responder
receives the test frames, updates its state table, and processes the
test frames as required by the given measurement procedure (e.g., only
counts them for the throughput test, handles timestamps for latency or
PDV tests, etc.).</t>
<t>If the Responder sends test frames, then it uses the four tuples
from its state table. The reading order of the state table may follow
different policies (discussed in <xref target="st_wr_order"
format="default"/>). The Initiator receives the test frames and
processes them as required by the given measurement procedure.</t>
<t>As for the actual measurement procedures, the usage of the updated
ones from <xref target="RFC8219" sectionFormat="of" section="7"/> is
<bcp14>RECOMMENDED</bcp14>.</t>
</section>
<section anchor="meas_conn_tear_down_rate" numbered="true" toc="default">
<name>Measurement of the Connection Tear-Down Rate</name>
<t>Connection tear-down can cause significant load for the NATxy
gateway. The connection tear-down performance can be measured as
follows:</t>
<ol spacing="normal" type="1">
<li>Load a certain number of connections (N) into the connection
tracking table of the DUT (in the same way as done to measure the
maximum connection establishment rate).</li>
<li>Record TimestampA.</li>
<li>Delete the content of the connection tracking table of the DUT.</l
i>
<li>Record TimestampB.</li>
</ol>
<t>The connection tear-down rate can be computed as:</t>
<t indent="5">connection tear-down rate = N / ( TimestampB - TimestampA)
</t>
<t>The connection tear-down rate <bcp14>SHOULD</bcp14> be measured for
various values of N.</t>
<t>It is assumed that the content of the connection tracking table may
be deleted by an out-of-band control mechanism specific to the given
NATxy gateway implementation (e.g., by removing the appropriate kernel
module under Linux).</t>
<t>It is noted that the performance of removing the entire content of
the connection tracking table at one time may be different from
removing all the entries one by one.</t>
</section>
<section anchor="meas_contr_capacity" numbered="true" toc="default">
<name>Measurement of the Connection Tracking Table Capacity</name>
<t>The connection tracking table capacity is an important metric of
stateful NATxy gateways. Its measurement is not easy, because an
elementary step of a validated maximum connection establishment rate
measurement (defined in <xref target="validation_of_conn"
format="default"/>) may have only a few distinct observable outcomes,
but some of them may have different root causes:</t>
<ul spacing="normal">
<li>
<t>During test phase 1, the number of test frames received by the
Responder is less than the number of test frames sent by the
Initiator. It may have different root causes, including:</t>
<ul spacing="normal">
<li>
<t>The R frame sending rate was higher than the maximum
connection establishment rate. (Note that now the maximum
connection establishment rate is considered unknown because
one cannot measure the maximum connection establishment
without <xref target="assumption1" format="none">assumption 1</x
ref> in <xref target="ctrl_conntrack"
format="default"/>.) This root cause may be eliminated by
lowering the R rate and re-executing the test. (This step may
be performed multiple times while R&gt;0.)</t>
</li>
<li>
<t>The capacity of the connection tracking table of the DUT
has been exhausted (and either the DUT does not want to
delete connections or the deletion of the connections makes it
slower; this case is not investigated further in test phase
1).</t>
</li>
</ul>
</li>
<li>
<t>During test phase 1, the number of test frames received by the
Responder equals the number of test frames sent by the Initiator.
In this case, the connections are validated in test phase 1. The
validation may have two kinds of observable results:</t>
<ol spacing="normal" type="1">
<li>
<t>The number of validation frames received by the Initiator
equals the number of validation frames sent by the Responder.
(It proves that the capacity of the connection tracking table
of the DUT is enough and both R and r were chosen
properly.)</t>
</li>
<li>
<t>The number of validation frames received by the Initiator
is less than the number of validation frames sent by the
Responder. This phenomenon may have various root causes:</t>
<ul spacing="normal">
<li>
<t>The capacity of the connection tracking table of the
DUT has been exhausted. (It does not matter whether some
existing connections are discarded and new ones are
stored or if the new connections are discarded. Some
connections are lost anyway, and it makes validation
fail.)</t>
</li>
<li>
<t>The R frame sending rate used by the Initiator was too
high in test phase 1; thus, some connections were not
established even though all test frames arrived at the
Responder. This root cause may be eliminated by lowering
the R rate and re-executing the test. (This step may be
performed multiple times while R&gt;0.)</t>
</li>
<li>
<t>The r frame sending rate used by the Responder was too
high in test phase 2; thus, some test frames did not
arrive at the Initiator even though all connections were
present in the connection tracking table of the DUT. This
root cause may be eliminated by lowering the r rate and
re-executing the test. (This step may be performed
multiple times while r&gt;0.)</t>
</li>
</ul>
<t>This is the problem: As the above three root causes are
indistinguishable, it is not easy to decide whether R or r
should be decreased.</t>
</li>
</ol>
</li>
</ul>
<t>Experience shows that the DUT may collapse if its memory is
exhausted. Such a situation may make the connection tracking table
capacity measurements rather inconvenient. This possibility is
included in the recommended measurement procedure, but the detection
and elimination of such a situation is not addressed (e.g., how the
algorithm can reset the DUT).</t>
<t>For the connection tracking table size measurement, first, one needs
a safe number: C0. It is a precondition that C0 number of connections
can surely be stored in the connection tracking table of the
DUT. Using C0, one can determine the maximum connection establishment
rate using C0 number of connections. It is done with a binary search
using validation. The result is R0. The values C0 and R0 will serve as
"safe" starting values for the following two searches.</t>
<t>First, an exponential search is performed to find the order of
magnitude of the connection tracking table capacity. The search stops
if the DUT collapses OR the maximum connection establishment rate
severely drops (e.g., to its one tenth) due to doubling the number of
connections.</t>
<t>Then, the result of the exponential search gives the order of
magnitude of the size of the connection tracking table. Before
disclosing the possible algorithms to determine the exact size of the
connection tracking table, three possible replacement policies for the
NATxy gateway are considered:</t>
<ol spacing="normal" type="1">
<li>
<t>The gateway does not delete any live connections until their time
out expires.</t>
</li>
<li>
<t>The gateway replaces the live connections according to the Least
Recently Used (LRU) policy.</t>
</li>
<li>
<t>The gateway does a garbage collection when its connection
tracking table is full and a frame with a new four tuple
arrives. During the garbage collection, it deletes the K LRU connect
ions, where K is greater than 1.</t>
</li>
</ol>
<t>Now, it is examined what happens and how many validation frames
arrive in the three cases. Let the size of the connection tracking
table be S and the number of preliminary frames be N, where S is less
than N.</t>
<ol spacing="normal" type="1">
<li>
<t>The connections defined by the first S test frames are
registered into the connection tracking table of the DUT, and
the last N-S connections are lost. (It is another question if the
last N-S test frames are translated and forwarded in test phase 1
or simply dropped.) During validation, the validation frames with
four tuples corresponding to the first S test frames will arrive
at the Initiator and the other N-S validation frames will be
lost.</t>
</li>
<li>
<t>All connections are registered into the connection tracking
table of the DUT, but the first N-S connections are replaced (and
thus lost). During validation, the validation frames with four
tuples corresponding to the last S test frames will arrive to the
Initiator, and the other N-S validation frames will be lost.</t>
</li>
<li>
<t>Depending on the values of K, S, and N, maybe less than S
connections will survive. In the worst case, only S-K+1
validation frames arrive, even though the size of the connection
tracking table is S.</t>
</li>
</ol>
<t>If one knows that the stateful NATxy gateway uses the first or
second replacement policy and one also knows that both R and r rates
are low enough, then the final step of determining the size of the
connection tracking table is simple. If the Responder sent N
validation frames and the Initiator received N' of them, then the size
of the connection tracking table is N'.</t>
<t>In the general case, a binary search is performed to find the exact
value of the connection tracking table capacity within E error. The
search chooses the lower half of the interval if the DUT collapses OR
the maximum connection establishment rate severely drops (e.g., to its
half); otherwise, it chooses the higher half. The search stops if the
size of the interval is less than the E error.</t>
<t>The algorithms for the general case are defined using C, like the
pseudocode in <xref target="meas_contr_capacity_algo"
format="default"/>. In practice, this algorithm may be made more
efficient in the way that the binary search for the maximum connection
establishment rate stops if an elementary test fails at a rate under
RS*beta or RS*gamma during the external search or during the final
binary search for the capacity of the connection tracking table,
respectively. (This saves a high amount of execution time by
eliminating the long-lasting tests at low rates.)
</t>
<figure anchor="meas_contr_capacity_algo">
<name>Measurement of the Connection Tracking Table Capacity</name>
<sourcecode type="pseudocode"><![CDATA[
// The binarySearchForMaximumConnectionCstablishmentRate(c,r) // The binarySearchForMaximumConnectionCstablishmentRate(c,r)
// function performs a binary search for the maximum connection // function performs a binary search for the maximum connection
// establishment rate in the [0, r] interval using c number of // establishment rate in the [0, r] interval using c number of
// connections. // connections.
// This is an exponential search for finding the order of magnitude // This is an exponential search for finding the order of magnitude
// of the connection tracking table capacity // of the connection tracking table capacity
// Variables: // Variables:
// C0 and R0 are beginning safe values for the connection // C0 and R0 are beginning safe values for the connection
// tracking table size and connection establishment rate, // tracking table size and connection establishment rate,
// respectively // respectively
// CS and RS are their currently used safe values // CS and RS are their currently used safe values
// CT and RT are their values for the current examination // CT and RT are their values for the current examination
// beta is a factor expressing an unacceptable drop in R (e.g. // beta is a factor expressing an unacceptable drop in R (e.g.,
// beta=0.1) // beta=0.1)
// maxrate is the maximum frame rate for the media // maxrate is the maximum frame rate for the media
R0=binarySearchForMaximumConnectionCstablishmentRate(C0,maxrate); R0=binarySearchForMaximumConnectionCstablishmentRate(C0,maxrate);
for ( CS=C0, RS=R0; 1; CS=CT, RS=RT ) for ( CS=C0, RS=R0; 1; CS=CT, RS=RT )
{ {
CT=2*CS; CT=2*CS;
RT=binarySearchForMaximumConnectionCstablishmentRate(CT,RS); RT=binarySearchForMaximumConnectionCstablishmentRate(CT,RS);
if ( DUT_collapsed || RT < RS*beta ) if ( DUT_collapsed || RT < RS*beta )
break; break;
} }
// At this point, the size of the connection tracking table is // At this point, the size of the connection tracking table is
// between CS and CT. // between CS and CT.
// This is the final binary search for finding the connection // This is the final binary search for finding the connection
// tracking table capacity within E error // tracking table capacity within E error
// Variables: // Variables:
// CS and RS are the safe values for connection tracking table size // CS and RS are the safe values for connection tracking table size
// and connection establishment rate, respectively // and connection establishment rate, respectively
// C and R are the values for the current examination // C and R are the values for the current examination
// gamma is a factor expressing an unacceptable drop in R // gamma is a factor expressing an unacceptable drop in R
// (e.g. gamma=0.5) // (e.g., gamma=0.5)
for ( D=CT-CS; D>E; D=CT-CS ) for ( D=CT-CS; D>E; D=CT-CS )
{ {
C=(CS+CT)/2; C=(CS+CT)/2;
R=binarySearchForMaximumConnectionCstablishmentRate(C,RS); R=binarySearchForMaximumConnectionCstablishmentRate(C,RS);
if ( DUT_collapsed || R < RS*gamma ) if ( DUT_collapsed || R < RS*gamma )
CT=C; // take the lower half of the interval CT=C; // take the lower half of the interval
else else
CS=C,RS=R; // take the upper half of the interval CS=C,RS=R; // take the upper half of the interval
} }
// At this point, the size of the connection tracking table is // At this point, the size of the connection tracking table is
// CS within E error. // CS within E error.
]]></sourcecode> ]]></sourcecode>
<postamble></postamble>
</figure> </figure>
<t keepWithPrevious="true"/>
</section>
</section> <section anchor="st_wr_order" numbered="true" toc="default">
<name>Writing and Reading Order of the State Table</name>
<section anchor="st_wr_order" title="Writing and Reading Order of the S <t>As for the writing policy of the state table of the Responder,
tate Table"> round robin is <bcp14>RECOMMENDED</bcp14>, because it ensures that its
<t>As for the writing policy of the state table of the Responder, entries are automatically kept fresh and consistent with that of the
round robin is RECOMMENDED, connection tracking table of the DUT.
because it ensures that its entries are automatically kept fresh </t>
and consistent with <t>The Responder can read its state table in various orders, for
that of the connection tracking table of the DUT. example:
</t> </t>
<t>The Responder can read its state table in various orders, for <ul spacing="normal">
example: <li>
<list style="symbols"> <t>pseudorandom</t>
<t>pseudorandom</t> </li>
<t>round-robin</t> <li>
</list> <t>round robin</t>
</t> </li>
<t> </ul>
Pseudorandom is RECOMMENDED to follow the approach of <xref targe <t>Pseudorandom is <bcp14>RECOMMENDED</bcp14> to follow the approach
t="RFC4814"/>. of <xref target="RFC4814" format="default"/>. Round robin may be used
Round-robin may be used as a computationally cheaper alternative. as a computationally cheaper alternative.
</t> </t>
</section> </section>
</section> </section>
<section anchor="meas_scalability" numbered="true" toc="default">
<name>Scalability Measurements</name>
<section anchor="meas_scalability" title="Scalability Measurements"> <!--[rfced] May we clarify the singular/plural usage in this sentence as
<t>As for scalability measurements, no new types of performance metrics follows??
are defined,
but it is RECOMMENDED to perform measurement series through which the v
alue of one or more parameter(s)
is/are changed to discover how the various values of the given paramete
r(s) influence
the performance of the DUT.
</t>
<section anchor="sc_net_flows" title="Scalability Against the Number of Network Flows"> Original:
<t>The scalability measurements aim to quantify how the performance ...but it is RECOMMENDED to perform measurement series
of the stateful NATxy gateways degrades with the increase of the through which the value of one or more parameter(s) is/are changed to
number of discover how the various values of the given parameter(s) influence
network flows.</t> the performance of the DUT.
<t>As for the actual values for the number of network flows to be Perhaps:
used during
the measurement series, it is RECOMMENDED to use some representat
ive values from
the range of the potential number of network flows the DUT may be
faced with
during its intended usage.</t>
<t>It is important, how the given number of network flows are gen ...but it is RECOMMENDED to perform measurement series
erated. The sizes through which the value of each parameter is changed to
of the ranges of the source and destination IP addresses and port discover how the various values of the each given parameter influences
numbers are the performance of the DUT.
essential parameters to be reported together with the results. Pl
ease see also
<xref target="reporting_format"/> about the reporting format.</t>
<t>If a single IP address pair is used, then it is RECOMMENDED to -->
use
<list style="symbols"> <t>As for scalability measurements, no new types of performance metrics
<t>a fixed, larger source port number range (e.g., a few times are defined, but it is <bcp14>RECOMMENDED</bcp14> to perform measurement
10,000)</t> series through which the value of one or more parameter(s) are
<t>a variable size destination port number range (e.g. 10; 100; changed to discover how the various values of the given parameter(s)
1,000; etc.), influence the performance of the DUT.
where its expedient granularity depends on the purpose.</t> </t>
</list> <section anchor="sc_net_flows" numbered="true" toc="default">
<name>Scalability Against the Number of Network Flows</name>
<t>The scalability measurements aim to quantify how the performance of
the stateful NATxy gateways degrades with the increase of the number
of network flows.</t>
<t>As for the actual values for the number of network flows to be used
during the measurement series, it is <bcp14>RECOMMENDED</bcp14> to use
some representative values from the range of the potential number of
network flows the DUT may be faced with during its intended usage.</t>
<t>It is important how the given number of network flows are
generated. The sizes of the ranges of the source and destination IP
addresses and port numbers are essential parameters to be reported
together with the results. Please also see <xref
target="reporting_format" format="default"/> about the reporting
format.</t>
<t>If a single IP address pair is used, then it is <bcp14>RECOMMENDED</b
cp14> to use:
</t> </t>
<ul spacing="normal">
<li>
<t>a fixed, larger source port number range (e.g., a few times
10,000) and</t>
</li>
<li>
<t>a variable-size destination port number range (e.g., 10, 100,
1,000, etc.), where its expedient granularity depends on the
purpose.</t>
</li>
</ul>
</section> </section>
<section anchor="sc_cpu_cores" numbered="true" toc="default">
<section anchor="sc_cpu_cores" title="Scalability Against the Number of <name>Scalability Against the Number of CPU Cores</name>
CPU Cores"> <t>Stateful NATxy gateways are often implemented in software that is
not bound to a specific hardware but can be executed by commodity
<t>Stateful NATxy gateways are often implemented in software that are servers. To facilitate the comparison of their performance, it can be
not bound useful to determine:
to a specific hardware but can be executed by commodity servers. </t>
To facilitate <ul spacing="normal">
the comparison of their performance, it can be useful to determin <li>
e <t>the performance of the various implementations using a single
<list style="symbols"> core of a well-known CPU and</t>
<t>the performance of the various implementations using a single co </li>
re of a well-known CPU</t> <li>
<t>the scale-up of the performance of the various implementatio <t>the scale-up of the performance of the various implementations
ns with the number of CPU cores.</t> with the number of CPU cores.</t>
</list> </li>
</t> </ul>
<t>If the number of the available CPU cores is a power of two, then it
<t>If the number of the available CPU cores is a power of two, then i is <bcp14>RECOMMENDED</bcp14> to perform the tests with 1, 2, 4, 8,
t is RECOMMENDED 16, etc. number of active CPU cores of the DUT.</t>
to perform the tests with 1, 2, 4, 8, 16, etc. number of active C
PU cores of the DUT.</t>
</section> </section>
</section> </section>
<section anchor="reporting_format" title="Reporting Format"> <section anchor="reporting_format" numbered="true" toc="default">
<name>Reporting Format</name>
<t>Measurements MUST be executed multiple times. The necessary number o <t>Measurements <bcp14>MUST</bcp14> be executed multiple times. The
f repetitions necessary number of repetitions to achieve statistically reliable
to achieve statistically reliable results may depend on the consistent results may depend on the consistent or scattered nature of the results.
or scattered nature of the results. The report of the results <bcp14>MUST</bcp14> contain the number of
The report of the results MUST contain the number of repetitions of the repetitions of the measurements. The median is <bcp14>RECOMMENDED</bcp14>
measurements. as the summarizing function of the results complemented with the first
Median is RECOMMENDED as the summarizing function of the results comple percentile and the 99th percentile as indices of the dispersion of the
mented with the results. The average and standard deviation <bcp14>MAY</bcp14> also be
first percentile and the 99th percentile as indices of the dispersion o reported.
f the results. </t>
Average and standard deviation MAY also be reported. <t>All parameters and settings that may influence the performance of the
</t> DUT <bcp14>MUST</bcp14> be reported. Some of them may be specific to the
given NATxy gateway implementation, like the "hashsize" (hash table
<t>All parameters and settings that may influence the performance of th size) and "nf_conntrack_max" (number of connection tracking table
e DUT MUST be entries) values for iptables or the limit of the number of states for
reported. Some of them may be specific to the given NATxy gateway imple OpenBSD PF (set by the "set limit states number" command in the pf.conf
mentation, like the file).
"hashsize" (hash table size) and "nf_conntrack_max" (number of connecti </t>
on tracking <t keepWithNext="true"/>
table entries) values for iptables or the limit of the number of states
for OpenBSD PF
(set by the "set limit states number" command in the pf.conf file).
</t>
<figure anchor="iptables-conn-scale" align="center" title="Example table:
Maximum connection establishment rate of iptables against the number of s
essions">
<preamble></preamble>
<artwork align="left"><![CDATA[
number of sessions (req.) 0.4M 4M 40M 400M
source port numbers (req.) 40,000 40,000 40,000 40,000
destination port numbers (req.) 10 100 1,000 10,000
"hashsize" (i.s.) 2^17 2^20 2^23 2^27
"nf_conntrack_max" (i.s.) 2^20 2^23 2^26 2^30
num. sessions / "hashsize" (i.s.) 3.05 3.81 4.77 2.98
number of experiments (req.) 10 10 10 10
error of binary search (req.) 1,000 1,000 1,000 1,000
connections/s median (req.)
connections/s 1st perc. (req.)
connections/s 99th perc. (req.)
]]></artwork>
<postamble></postamble>
</figure>
<t><xref target="iptables-conn-scale"/> shows an example of table headi
ngs for
reporting the measurement results for the scalability of the iptables s
tateful NAT44
implementation against the number of sessions. The table indicates the
always required fields
(req.) and the implementation-specific ones (i.s.).
A computed value was also added in row 6; it is the number of sessions
per hashsize ratio, which
helps the reader to interpret the achieved maximum connection establish
ment rate.
(A lower value results in shorter linked lists hanging on the entries o
f the hash
table thus facilitating higher performance. The ratio is varying, becau
se the number of
sessions is always a power of 10, whereas the hash table size is a powe
r of 2.)
To reflect the accuracy of the results, the table contains the value of
the "error" of the
binary search, which expresses the stopping criterion for the binary se
arch. The binary
search stops, when the difference between the "higher limit" and "lower
limit" of the
binary search is less than or equal to "error".
</t>
<t> The table MUST be complemented with reporting the relevant paramete
rs of the
DUT. If the DUT is a general-purpose computer and some software NATxy g
ateway implementation is tested,
then the hardware description SHOULD include: computer type, CPU type,
and number of active CPU cores,
memory type, size and speed, network interface card type (reflecting al
so the speed),
the fact that direct cable connections were used or the type of the swi
tch used for
interconnecting the Tester and the DUT. Operating system type and versi
on, kernel version, and the version
of the NATxy gateway implementation (including last commit date and num
ber if applicable) SHOULD also be given.
</t>
</section>
<section anchor="impl_exp" title="Implementation and Experience"> <table anchor="iptables-conn-scale" align="left">
<t>The stateful extension of siitperf <xref target="SIITPERF"/> is an i <name>Example Table of the Maximum Connection Establishment Rate of
mplementation of this concept. Iptables Against the Number of Sessions</name>
Its first version only supporting multiple port numbers is documented i <tbody>
n this (open access) paper <xref target="LEN2022"/>. <tr>
Its extended version also supporting multiple IP addresses is documente <td align="left">number of sessions (req.)</td>
d in this (open access) paper <xref target="LEN2024b"/>. <td align="right">0.4M</td>
</t> <td align="right">4M</td>
<t> The proposed benchmarking methodology has been validated by perform <td align="right">40M</td>
ing <td align="right">400M</td>
benchmarking measurements with three radically different stateful NAT64 </tr>
implementations (Jool, tayga+iptables, OpenBSD PF) in (open access) pap <tr>
er <td align="left">source port numbers (req.)</td>
<xref target="LEN2023"/>. <td align="right">40,000</td>
</t> <td align="right">40,000</td>
<t>Further experience with this methodology using siitperf for measurin <td align="right">40,000</td>
g the <td align="right">40,000</td>
scalability of the iptables stateful NAT44 and Jool stateful NAT64 </tr>
implementations are described in <tr>
<xref target="I-D.lencse-v6ops-transition-scalability"/>. <td align="left">destination port numbers (req.)</td>
</t> <td align="right">10</td>
<t>This methodology was successfully applied for the benchmarking of va <td align="right">100</td>
rious <td align="right">1,000</td>
IPv4aas (IPv4-as-a-Service) technologies without the usage of technolog <td align="right">10,000</td>
y-specific </tr>
Testers by reducing the aggregate of their CE (Customer Edge) and PE (P <tr>
rovider Edge) <td align="left">"hashsize" (i.s.)</td>
devices to a stateful NAT44 gateway documented in (open access) paper < <td align="right">2<sup>17</sup></td>
xref target="LEN2024a"/>. <td align="right">2<sup>20</sup></td>
</t> <td align="right">2<sup>23</sup></td>
</section> <td align="right">2<sup>27</sup></td>
</tr>
<tr>
<td align="left">"nf_conntrack_max" (i.s.)</td>
<td align="right">2<sup>20</sup></td>
<td align="right">2<sup>23</sup></td>
<td align="right">2<sup>26</sup></td>
<td align="right">2<sup>30</sup></td>
</tr>
<tr>
<td align="left">num. sessions / "hashsize" (i.s.)</td>
<td align="right">3.05</td>
<td align="right">3.81</td>
<td align="right">4.77</td>
<td align="right">2.98</td>
</tr>
<tr>
<td align="left">number of experiments (req.)</td>
<td align="right">10</td>
<td align="right">10</td>
<td align="right">10</td>
<td align="right">10</td>
</tr>
<tr>
<td align="left">error of binary search (req.)</td>
<td align="right">1,000</td>
<td align="right">1,000</td>
<td align="right">1,000</td>
<td align="right">1,000</td>
</tr>
<tr>
<td align="left">connections/s median (req.)</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
<tr>
<td align="left">connections/s 1st perc. (req.)</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
<tr>
<td align="left">connections/s 99th perc. (req.)</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
</tbody>
</table>
<section anchor="udp_or_tcp" title="Limitations of using UDP as Transport La <t keepWithPrevious="true"/>
yer Protocol">
<t>The test frame format defined in RFC 2544 exclusively uses UDP (and
not TCP) as a
transport layer protocol. Testing with UDP was kept in both RFC 5180 an
d RFC 8219 regarding
the standard benchmarking procedures (throughput, latency, frame loss r
ate, etc.).
The benchmarking methodology proposed in this document follows this lon
g established
benchmarking tradition using UDP as a transport layer protocol, too. Th
e rationale for this
is that the standard benchmarking procedures require sending frames at
arbitrary constant
frame rates, which would violate the flow control and congestion contro
l algorithms of the
TCP protocol. TCP connection setup (using the three-way handshake) woul
d further complicate testing.
</t>
<t>Further potential transport layer protocols e.g., DCCP <xref target= <t><xref target="iptables-conn-scale" format="default"/> shows an
"RFC4340"/> and SCTP <xref target="RFC9260"/> example of table headings for reporting the measurement results regarding
are outside of the scope of this document, as the widely-used the
stateful NAT44 and stateful NAT64 implementations do not support them. scalability of the iptables stateful NAT44 implementation against the
Although QUIC <xref target="RFC9000"/> number of sessions. The table indicates the required fields
is also considered a transport layer protocol, but QUIC packets are car (req.) and the implementation-specific ones (i.s.). A computed value
ried in UDP was also added in row 6; it is the number of sessions per hashsize
datagrams thus QUIC does not need a special handling. ratio, which helps the reader to interpret the achieved maximum
</t> connection establishment rate. (A lower value results in shorter linked
lists hanging on the entries of the hash table, thus facilitating higher
performance. The ratio is varying, because the number of sessions is
always a power of 10, whereas the hash table size is a power of 2.) To
reflect the accuracy of the results, the table contains the value of the
"error" of the binary search, which expresses the stopping criterion for
the binary search. The binary search stops when the difference between
the "higher limit" and "lower limit" of the binary search is less than
or equal to the "error".
<t>Some stateful NATxy solutions handle TCP and UDP differently, e.g. i </t>
ptables uses 30s <t>The table <bcp14>MUST</bcp14> be complemented with reporting the
timeout for UDP and 60s timeout for TCP. Thus benchmarking results prod relevant parameters of the DUT. If the DUT is a general-purpose computer
uced using UDP do not and some software NATxy gateway implementation is tested, then the
necessarily characterize the performance of a NATxy gateway well enough wh hardware description <bcp14>SHOULD</bcp14> include: the computer type, CPU
en they type and number of active CPU cores, memory type, size and speed,
are used for forwarding Internet traffic. As for the given example, tim network interface card type (also reflecting the speed), the fact that
eout values of the DUT may direct cable connections were used, and the type of the switch used for
be adjusted, but it requires extra consideration. interconnecting the Tester and the DUT. The operating system type and
</t> version, kernel version, and version of the NATxy gateway
<t>Other differences in handling UDP or TCP are also possible. Thus, th implementation (including the last commit date and number if applicable)
e authors recommend that <bcp14>SHOULD</bcp14> also be given.
further investigations should be performed in this field. </t>
</t>
<t>As a mitigation of this problem, this document recommends that testi
ng with protocols using TCP
(like HTTP and HTTPS up to version 2) can be performed as described in
<xref target="RFC9411"/>.
This approach also solves the potential problem of protocol helpers may
be present in the stateful DUT.
</t>
<t>As for HTTP/3, it uses QUIC, which uses UDP as stated above. It shou
ld be noted that QUIC is treated
as any other UDP payload. The proposed measurement method does not aim
to measure the performance
of QUIC, rather it aims to measure the performance of the stateful NATx
y gateway.
</t>
</section> </section>
<section anchor="Acknowledgements" title="Acknowledgements"> <section anchor="impl_exp" numbered="true" toc="default">
<t>The authors would like to thank Al Morton, Sarah Banks, Edwin Cordeiro, <name>Implementation and Experience</name>
Lukasz Bromirski,
Sándor Répás, Tamás Hetényi, Timothy Winters, Eduard Vasilenko, Minh Ng
oc Tran, Paolo Volpato,
Zeqi Lai, and Bertalan Kovács for their comments.</t>
<t>The authors thank Warren Kumari, Michael Scharf, Alexey Melnikov, Ro
bert Sparks, David Dong,
Roman Danyliw, Erik Kline, Murray Kucherawy, Zaheduzzaman Sarker, and É
ric Vyncke
for their reviews and comments.</t>
<t>This work was supported by the Japan Trust International Research Coo
peration Program
of the National Institute of Information and Communications Technology (
NICT), Japan.</t>
</section>
<!-- Possibly a 'Contributors' section ... -->
<section anchor="IANA" title="IANA Considerations">
<t>This document does not make any request to IANA.</t>
</section>
<section anchor="Security" title="Security Considerations"> <t>The stateful extension of siitperf <xref target="SIITPERF"
<t>This document has no further security considerations beyond that of <xre format="default"/> is an implementation of this concept. Its first
f target="RFC8219"/>. version that only supports multiple port numbers is documented in this
They should be cited here so that they be applied not only for the (open access) paper: <xref target="LEN2022" format="default"/>. Its
benchmarking of IPv6 transition technologies but also for the benchmarki extended version that also supports multiple IP addresses is documented in
ng of any this (open access) paper: <xref target="LEN2024b" format="default"/>.
stateful NATxy gateways (allowing for x=y, too).</t> </t>
</section>
</middle>
<!-- *****BACK MATTER ***** --> <t>The proposed benchmarking methodology has been validated by
performing benchmarking measurements with three radically different
stateful NAT64 implementations (Jool, tayga+iptables, and OpenBSD PF) in t
his
(open access) paper: <xref target="LEN2023" format="default"/>.</t>
<back> <t>Further experience with this methodology of using siitperf for measurin
<!-- References split into informative and normative --> g
the scalability of the iptables stateful NAT44 and Jool stateful NAT64
implementations are described in <xref
target="I-D.lencse-v6ops-transition-scalability" format="default"/>.</t>
<!-- There are 2 ways to insert reference entries from the citation libraries <t>This methodology was successfully applied for the benchmarking of
: various IPv4-as-a-Service (IPv4aas) technologies without the usage of
1. define an ENTITY at the top, and use "ampersand character"RFC2629; here ( technology-specific Testers by reducing the aggregate of their Customer
as shown) Edge (CE) and Provider Edge (PE) devices to a stateful NAT44 gateway
2. simply use a PI "less than character"?rfc include="reference.RFC.2119.xml documented in this (open access) paper: <xref target="LEN2024a"
"?> here format="default"/>.</t>
(for I-Ds: include="reference.I-D.narten-iana-considerations-rfc2434bis.x </section>
ml")
Both are cited textually in the same manner: by using xref elements. <section anchor="udp_or_tcp" numbered="true" toc="default">
If you use the PI option, xml2rfc will, by default, try to find included fil <name>Limitations of Using UDP as a Transport Layer Protocol</name>
es in the same
directory as the including file. You can also define the XML_LIBRARY environ
ment variable
with a value containing a set of directories to search. These can be either
in the local
filing system or remote ones accessed by http (http://domain/dir/... ).-->
<references title="Normative References"> <t>The test frame format defined in <xref target="RFC2544"/> exclusively u
<!--?rfc include="http://xml.resource.org/public/rfc/bibxml/reference.RFC.21 ses UDP (and
19.xml"?--> not TCP) as a transport layer protocol. Testing with UDP was kept in
both <xref target="RFC5180"/> and <xref target="RFC8219"/> regarding the s
tandard benchmarking
procedures (throughput, latency, frame loss rate, etc.). The
benchmarking methodology proposed in this document follows this long-estab
lished benchmarking tradition using UDP as a transport layer
protocol, too. The rationale for this is that the standard benchmarking
procedures require sending frames at arbitrary constant frame rates,
which would violate the flow control and congestion control algorithms
of the TCP protocol. TCP connection setup (using the three-way
handshake) would further complicate testing.</t>
&RFC2119; <t>Further potential transport layer protocols, e.g., the Datagram Congest
&RFC1918; ion Control Protocol (DCCP) <xref
&RFC2544; target="RFC4340" format="default"/> and the Stream Control Transmission Pr
&RFC3022; otocol (SCTP) <xref target="RFC9260"
&RFC4340; format="default"/>, are outside of the scope of this document, as the
&RFC4814; widely used stateful NAT44 and stateful NAT64 implementations do not
&RFC5180; support them. Although QUIC <xref target="RFC9000" format="default"/> is
&RFC6146; also considered a transport layer protocol, QUIC packets are carried
&RFC7599; in UDP datagrams; thus, QUIC does not need a special handling.</t>
&RFC8174;
&RFC8219;
&RFC9000;
&RFC9260;
&RFC9411;
</references> <t>Some stateful NATxy solutions handle TCP and UDP differently,
e.g., iptables use a 30s timeout for UDP and a 60s timeout for TCP. Thus,
benchmarking results produced using UDP do not necessarily characterize
the performance of a NATxy gateway well enough when they are used for
forwarding Internet traffic. As for the given example, timeout values of
the DUT may be adjusted, but it requires extra consideration.</t>
<references title="Informative References"> <t>Other differences in handling UDP or TCP are also possible. Thus, the
<!-- Here we use entities that we defined at the beginning. --> authors recommend that further investigations should be performed in
this field.</t>
<?rfc include='reference.I-D.lencse-v6ops-transition-scalability'?> <t>As a mitigation of this problem, this document recommends that
testing with protocols using TCP (like HTTP and HTTPS up to version 2)
can be performed as described in <xref target="RFC9411"
format="default"/>. This approach also solves the potential problem of
protocol helpers that may be present in the stateful DUT.</t>
<reference anchor="DUST1964" <t>As for HTTP/3, it uses QUIC, which uses UDP as stated above. It
target="https://dl.acm.org/doi/10.1145/364520.364540"> should be noted that QUIC is treated as any other UDP payload. The
<front> proposed measurement method does not aim to measure the performance of
<title>Algorithm 235: Random permutation QUIC, rather, it aims to measure the performance of the stateful NATxy
</title> gateway.</t>
<author initials="R." surname="Durstenfeld"> </section>
<organization></organization>
</author>
<date day="" month="July" year="1964"/>
</front>
<seriesInfo name="" value="Communications of the ACM, vol. 7, no. 7, p.420
."/>
<seriesInfo name="DOI" value="10.1145/364520.364540"/>
</reference>
<reference anchor="IIR2020" <section anchor="IANA" numbered="true" toc="default">
target="https://www.iij.ad.jp/en/dev/iir/pdf/iir_vol49_report_EN.pdf"> <name>IANA Considerations</name>
<front> <t>This document has no IANA actions.</t>
<title>Periodic observation report: Internet trends as seen from IIJ inf </section>
rastructure - 2020
</title>
<author initials="T." surname="Kurahashi"> <section anchor="Security" numbered="true" toc="default">
<organization></organization> <name>Security Considerations</name>
</author> <t>This document has no further security considerations beyond that of
<author initials="Y." surname="Matsuzaki"> <xref target="RFC8219" format="default"/>. They should be cited here so
<organization></organization> that they can be applied not only for the benchmarking of IPv6 transition
</author> technologies but also for the benchmarking of any stateful NATxy
<author initials="T." surname="Sasaki"> gateways (allowing for x=y, too).</t>
<organization></organization> </section>
</author> </middle>
<author initials="T." surname="Saito"> <back>
<organization></organization>
</author>
<author initials="F." surname="Tsutsuji">
<organization></organization>
</author>
<date day="" month="Dec" year="2020"/>
</front>
<seriesInfo name="" value="Internet Infrastructure Review, vol. 49"/>
</reference>
<reference anchor="LEN2015" <displayreference target="I-D.lencse-v6ops-transition-scalability" to="SCALAB
target="http://www.hit.bme.hu/~lencse/publications/e98-b_8_1580.pdf"> ILITY"/>
<front> <references>
<title>Estimation of the Port Number Consumption of Web Browsing <name>References</name>
</title> <references>
<name>Normative References</name>
<author initials="G." surname="Lencse"> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.21
<organization></organization> 19.xml"/>
</author> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.1
918.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.2
544.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.3
022.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.4
340.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.4
814.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.5
180.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.6
146.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7
599.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8
174.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8
219.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9
000.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9
260.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9
411.xml"/>
</references>
<references>
<name>Informative References</name>
<date day="1" month="8" year="2015"/> <!-- [I-D.lencse-v6ops-transition-scalability] IESG state: Expired as of 06/19/2
</front> 4-->
<seriesInfo name="" value="IEICE Transactions on Communications, vol. E98- <xi:include href="https://datatracker.ietf.org/doc/bibxml3/draft-lencse-v
B, no. 8. pp. 1580-1588"/> 6ops-transition-scalability.xml"/>
<seriesInfo name="DOI" value="DOI: 10.1587/transcom.E98.B.1580"/>
</reference>
<reference anchor="LEN2020" <reference anchor="DUST1964" target="https://dl.acm.org/doi/pdf/10.1145/
target="http://ijates.org/index.php/ijates/article/view/291"> 364520.364540">
<front> <front>
<title>Adding RFC 4814 Random Port Feature to Siitperf: Design, Implemen <title>Algorithm 235: Random permutation
tation and Performance Estimation </title>
</title> <author initials="R." surname="Durstenfeld">
<organization/>
</author>
<date month="July" year="1964"/>
</front>
<refcontent>Communications of the ACM, vol. 7, no. 7, p. 420</refconte
nt>
<seriesInfo name="DOI" value="10.1145/364520.364540"/>
</reference>
<author initials="G." surname="Lencse"> <reference anchor="IIR2020" target="https://www.iij.ad.jp/en/dev/iir/pdf
<organization></organization> /iir_vol49_report_EN.pdf">
</author> <front>
<date day="" month="" year="2020"/> <title>Periodic Observation Report: Internet Trends as Seen from IIJ
</front> Infrastructure - 2020
<seriesInfo name="" value="International Journal of Advances in Telecommun </title>
ications, Electrotechnics, Signals and Systems, vol 9, no 3, pp. 18-26."/> <author initials="T." surname="Kurahashi">
<seriesInfo name="DOI" value="10.11601/ijates.v9i3.291"/> <organization/>
</reference> </author>
<author initials="Y." surname="Matsuzaki">
<organization/>
</author>
<author initials="T." surname="Sasaki">
<organization/>
</author>
<author initials="T." surname="Saito">
<organization/>
</author>
<author initials="F." surname="Tsutsuji">
<organization/>
</author>
<date month="December" year="2020"/>
</front>
<refcontent>Internet Initiative Japan Inc.</refcontent>
<refcontent>Internet Infrastructure Review, vol. 49</refcontent>
</reference>
<reference anchor="LEN2022" <reference anchor="LEN2015" target="https://www.hit.bme.hu/~lencse/publi
target="https://www.sciencedirect.com/science/article/pii/S0140366422001803" cations/e98-b_8_1580.pdf">
> <front>
<front> <title>Estimation of the Port Number Consumption of Web Browsing
<title>Design and Implementation of a Software Tester for Benchmarking S </title>
tateful NAT64xy Gateways: <author initials="G." surname="Lencse">
Theory and Practice of Extending Siitperf for Stateful Tests <organization/>
</title> </author>
<date month="August" year="2015"/>
</front>
<refcontent>IEICE Transactions on Communications, vol. E98-B, no. 8. p
p. 1580-1588</refcontent>
<seriesInfo name="DOI" value="10.1587/transcom.E98.B.1580"/>
</reference>
<author initials="G." surname="Lencse"> <reference anchor="LEN2020" target="http://ijates.org/index.php/ijates/a
<organization></organization> rticle/view/291">
</author> <front>
<title>Adding RFC 4814 Random Port Feature to Siitperf: Design, Impl
ementation and Performance Estimation
</title>
<author initials="G." surname="Lencse">
<organization/>
</author>
<date month="November" year="2020"/>
</front>
<refcontent>International Journal of Advances in Telecommunications, E
lectrotechnics, Signals and Systems, vol 9, no 3, pp. 18-26.</refcontent>
<seriesInfo name="DOI" value="10.11601/ijates.v9i3.291"/>
</reference>
<date day="1" month="August" year="2022"/> <reference anchor="LEN2022" target="https://www.sciencedirect.com/scienc
</front> e/article/pii/S0140366422001803">
<seriesInfo name="" value="Computer Communications, vol. 172, no. 1, pp. 7 <front>
5-88"/> <title>Design and Implementation of a Software Tester for Benchmarki
<seriesInfo name="DOI" value="10.1016/j.comcom.2022.05.028"/> ng Stateful NAT64xy Gateways: Theory and Practice of Extending Siitperf for Stat
</reference> eful Tests
</title>
<author initials="G." surname="Lencse">
<organization/>
</author>
<date month="August" year="2022"/>
</front>
<refcontent>Computer Communications, vol. 192, pp. 75-88</refcontent>
<seriesInfo name="DOI" value="10.1016/j.comcom.2022.05.028"/>
</reference>
<reference anchor="LEN2023" <reference anchor="LEN2023" target="https://www.sciencedirect.com/scienc
target="https://www.sciencedirect.com/science/article/pii/S0140366423002931" e/article/pii/S0140366423002931">
> <front>
<front> <title>Benchmarking methodology for stateful NAT64 gateways
<title>Benchmarking methodology for stateful NAT64 gateways </title>
</title> <author initials="G." surname="Lencse">
<organization/>
</author>
<author initials="K." surname="Shima">
<organization/>
</author>
<author initials="K." surname="Cho">
<organization/>
</author>
<date month="October" year="2023"/>
</front>
<refcontent>Computer Communications, vol. 210, pp. 256-272</refcontent
>
<seriesInfo name="DOI" value="10.1016/j.comcom.2023.08.009"/>
</reference>
<author initials="G." surname="Lencse"> <reference anchor="LEN2024a" target="https://www.sciencedirect.com/scien
<organization></organization> ce/article/pii/S0140366424000999">
</author> <front>
<author initials="K." surname="Shima"> <title>Benchmarking methodology for IPv4aaS technologies:
<organization></organization> Comparison of the scalability of the Jool implementation of 464XL
</author> AT and MAP-T
<author initials="K." surname="Cho"> </title>
<organization></organization> <author initials="G." surname="Lencse">
</author> <organization/>
</author>
<author initials="Á." surname="Bazsó">
<organization/>
</author>
<date month="April" year="2024"/>
</front>
<refcontent>Computer Communications, vol. 219, pp. 243-258</refcontent
>
<seriesInfo name="DOI" value="10.1016/j.comcom.2024.03.007"/>
</reference>
<date day="1" month="October" year="2023"/> <reference anchor="LEN2024b" target="https://www.sciencedirect.com/scien
</front> ce/article/abs/pii/S0140366424001993">
<seriesInfo name="" value="Computer Communications, vol. 210, no. 1, pp. 2 <front>
56-272"/> <title>Making stateless and stateful network performance measurement
<seriesInfo name="DOI" value="10.1016/j.comcom.2023.08.009"/> s unbiased
</reference> </title>
<author initials="G." surname="Lencse">
<organization/>
</author>
<date month="September" year="2024"/>
</front>
<refcontent>Computer Communications, vol. 225, pp. 141-155</refcontent
>
<seriesInfo name="DOI" value="10.1016/j.comcom.2024.05.018"/>
</reference>
<reference anchor="LEN2024a" <reference anchor="SIITPERF" target="https://github.com/lencsegabor/siit
target="https://www.sciencedirect.com/science/article/pii/S0140366424000999" perf">
> <front>
<front> <title>Siitperf: An RFC 8219 compliant SIIT and stateful NAT64/NAT44
<title>Benchmarking methodology for IPv4aaS technologies: tester
Comparison of the scalability of the Jool implementation of 464XL </title>
AT and MAP-T <author>
</title> <organization/>
</author>
<date month="September" year="2023"/>
</front>
<refcontent>commit 165cb7f</refcontent>
</reference>
</references>
</references>
<author initials="G." surname="Lencse"> <section anchor="Acknowledgements" numbered="false" toc="default">
<organization></organization> <name>Acknowledgements</name>
</author>
<author initials="Á." surname="Bazsó">
<organization></organization>
</author>
<date day="1" month="April" year="2024"/> <t>The authors would like to thank <contact fullname="Al Morton"/>,
</front> <contact fullname="Sarah Banks"/>, <contact fullname="Edwin Cordeiro"/>,
<seriesInfo name="" value="Computer Communications, vol. 219, no. 1, pp. 2 <contact fullname="Lukasz Bromirski"/>, <contact fullname="Sándor
43-258"/> Répás"/>, <contact fullname="Tamás Hetényi"/>, <contact
<seriesInfo name="DOI" value="10.1016/j.comcom.2024.03.007"/> fullname="Timothy Winters"/>, <contact fullname="Eduard Vasilenko"/>,
</reference> <contact fullname="Minh Ngoc Tran"/>, <contact fullname="Paolo
Volpato"/>, <contact fullname="Zeqi Lai"/>, and <contact
fullname="Bertalan Kovács"/> for their comments.</t>
<t>The authors thank <contact fullname="Warren Kumari"/>, <contact
fullname="Michael Scharf"/>, <contact fullname="Alexey Melnikov"/>,
<contact fullname="Robert Sparks"/>, <contact fullname="David Dong"/>,
<contact fullname="Roman Danyliw"/>, <contact fullname="Erik Kline"/>,
<contact fullname="Murray Kucherawy"/>, <contact fullname="Zaheduzzaman
Sarker"/>, and <contact fullname="Éric Vyncke"/> for their reviews and
comments.</t>
<t>This work was supported by the Japan Trust International Research
Cooperation Program of the National Institute of Information and
Communications Technology (NICT), Japan.</t>
</section>
</back>
<reference anchor="LEN2024b" <!-- [rfced] FYI - We have added expansions for abbreviations upon first use
target="https://www.sciencedirect.com/science/article/abs/pii/S0140366424001 per Section 3.6 of RFC 7322 ("RFC Style Guide"). Please review each
993"> expansion in the document carefully to ensure correctness.
<front>
<title>Making stateless and stateful network performance measurements un
biased
</title>
<author initials="G." surname="Lencse"> Border Relay (BR)
<organization></organization> Mapping of Address and Port using Translation (MAP-T)
</author> Datagram Congestion Control Protocol (DCCP)
Stream Control Transmission Protocol (SCTP)
<!-- <date day="1" month="April" year="2024"/> --> -->
</front>
<seriesInfo name="" value="Computer Communications"/>
<seriesInfo name="DOI" value="10.1016/j.comcom.2024.05.018"/>
</reference>
-&gt; <!-- [rfced] Please review the "Inclusive Language" portion of the online
<reference anchor="SIITPERF" Style Guide <https://www.rfc-editor.org/styleguide/part2/#inclusive_language>
target="https://github.com/lencsegabor/siitperf"> and let us know if any changes are needed. Updates of this nature typically
<front> result in more precise language, which is helpful for readers.
<title>Siitperf: An RFC 8219 compliant SIIT and stateful NAT64/NAT44 tes
ter written in C++ using DPDK
</title>
<author initials="G." surname="Lencse"> a. For example, please consider whether "black" should be updated.
<organization></organization>
</author>
<date day="" month="" year="2019-2023" /> b. In addition, please consider whether "tradition" and "traditional" should
</front> be updated for clarity. While the NIST website
<seriesInfo name="" value="source code"/> <https://www.nist.gov/nist-research-library/nist-technical-series-publications-a
<seriesInfo name="" value="available from GitHub"/> uthor-instructions#table1>
</reference> indicates that this term is potentially biased, it is also ambiguous.
<!-- --> "Tradition" is a subjective term, as it is not the same for everyone.
</references> -->
<section anchor="change_log" title="Change Log">
<section title="00">
<t>Initial version.
</t>
</section>
<section title="01">
<t>Updates based on the comments received on the BMWG mailing list and min
or corrections.
</t>
</section>
<section title="02">
<t><xref target="ctrl_conntrack"/> was completely re-written. As a consequ
ence,
the occurrences of the now undefined "mostly different" source port num
ber destination
port number combinations were deleted from <xref target="meas_max_conn_
est_rate"/>,
too.
</t>
</section>
<section title="03">
<t>Added <xref target="consider_stateful"/> about the consideration of the
cases of stateful operation.
</t>
<t>Consistency checking. Removal of some parts obsoleted by the previous r
e-writing
of <xref target="ctrl_conntrack"/>.
</t>
<t>Added <xref target="meas_conn_tear_down_rate"/> about the method for me
asuring connection tear-down rate.
</t>
<t>Updates for <xref target="impl_exp"/> about the implementation and expe
rience.
</t>
</section>
<section title="04">
<t>Update of the abstract.
</t>
<t>Added <xref target="validation_of_conn"/> about validation of connectio
n establishment.
</t>
<t>Added <xref target="meas_contr_capacity"/> about the method for measuri
ng connection tracking table capacity.
</t>
<t>Consistency checking and corrections.
</t>
</section>
<section title="00 - WG item">
<t>Added measurement setup for Stateful NAT64 gateways.
</t>
<t>Consistency checking and corrections.
</t>
</section>
<section title="01">
<t>Added Section 4.5.1 about typical types of measurement series and repor
ting format.
</t>
</section>
<section title="02">
<t>Added the usage of multiple IP addresses.</t>
<t>Section 4.5.1 was removed and split into two Sections:
<xref target="meas_scalability"/> about scalability measurements and
<xref target="reporting_format"/> about reporting format.
</t>
</section>
<section title="03">
<t>Updated the usage of multiple IP addresses.</t>
<t>Test phases were renamed as follows:
<list style="symbols">
<t>preliminary test phase --> test phase 1</t>
<t>real test phase --> test phase 2.</t>
</list>
</t>
</section>
<section title="04">
<t>Minor updates to <xref target="setup_term_multiple"/> and <xref target=
"impl_exp"/>.</t>
</section>
<section title="05">
<t>Minor updates addressing WGLC nits (adding the definition of "black box
", and
performing a high amount of grammatical corrections).</t>
</section>
<section title="06">
<t>Language editing addressing preliminary AD review comments by eliminati
ng the occurrences
of first person singular ("we", "our").</t>
</section>
<section title="07">
<t>Updates addressing IESG Last Call comments.</t>
</section>
</section>
</back>
</rfc> </rfc>
 End of changes. 146 change blocks. 
1603 lines changed or deleted 1604 lines changed or added

This html diff was produced by rfcdiff 1.48.