{"id":7106,"date":"2024-01-19T09:57:51","date_gmt":"2024-01-19T09:57:51","guid":{"rendered":"https:\/\/www.fibermall.com\/blog\/?p=7106"},"modified":"2024-03-15T10:24:34","modified_gmt":"2024-03-15T10:24:34","slug":"what-is-the-difference-between-infiniband-and-roce","status":"publish","type":"post","link":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm","title":{"rendered":"What is the Difference Between InfiniBand and RoCE?"},"content":{"rendered":"\n<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_76 ez-toc-wrap-left counter-hierarchy ez-toc-counter ez-toc-grey ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">Table of Contents<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#Data_Center_Network_Architecture\" >Data Center Network Architecture<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#Crossbar_architecture\" >Crossbar architecture<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#Clos_architecture\" >Clos architecture<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#Fat-Tree_architecture\" >Fat-Tree architecture<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#Spine-Leaf_network\" >Spine-Leaf network<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#InfiniBand\" >InfiniBand<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#RDMA_Remote_Direct_Memory_Access_protocol\" >RDMA (Remote Direct Memory Access) protocol<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#The_background_of_InfiniBand\" >The background of InfiniBand<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#The_development_history_of_InfiniBand\" >The development history of InfiniBand<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#InfiniBand_Network_Architecture\" >InfiniBand Network Architecture<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#RoCE\" >RoCE<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-12\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#The_birth_of_RoCE\" >The birth of RoCE<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-13\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#RoCE_V2\" >RoCE V2<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-14\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\/#Related_Posts\" >Related Posts<\/a><\/li><\/ul><\/li><\/ul><\/nav><\/div>\n<h2 class=\"wp-block-heading\" id=\"h-data-center-network-architecture\"><span class=\"ez-toc-section\" id=\"Data_Center_Network_Architecture\"><\/span><strong>Data Center Network Architecture<\/strong><strong><\/strong><span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"h-crossbar-architecture\"><span class=\"ez-toc-section\" id=\"Crossbar_architecture\"><\/span><strong>Crossbar architecture<\/strong><strong><\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li>A type of architecture derived from the earliest telephone switching network (crossbar switch)<\/li>\n\n\n\n<li>Consists of multiple input ports, multiple output ports, and a switch matrix<\/li>\n\n\n\n<li>Very flexible and efficient, can achieve arbitrary connections between different devices.<\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Crossbar-architecture.png\" alt=\"Crossbar architecture\" class=\"wp-image-7108\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Crossbar-architecture.png 991w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Crossbar-architecture-300x132.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Crossbar-architecture-768x339.png 768w\" sizes=\"(max-width: 991px) 100vw, 991px\" \/><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"h-clos-architecture\"><span class=\"ez-toc-section\" id=\"Clos_architecture\"><\/span><strong>Clos architecture<\/strong><strong><\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li>Born in 1952, proposed by a person named Charles Clos.<\/li>\n\n\n\n<li>Clos architecture mainly describes the structure of a multi-stage circuit switching network<\/li>\n\n\n\n<li>Clos architecture is an improvement of the crossbar structure, which can provide a non-blocking network. The advantage of Clos is that it saves costs and increases efficiency.<\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Clos-architecture.png\" alt=\"Clos architecture\" class=\"wp-image-7109\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Clos-architecture.png 779w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Clos-architecture-300x132.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Clos-architecture-768x338.png 768w\" sizes=\"(max-width: 779px) 100vw, 779px\" \/><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"h-fat-tree-architecture\"><span class=\"ez-toc-section\" id=\"Fat-Tree_architecture\"><\/span><strong>Fat-Tree architecture<\/strong><strong><\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>A Fat-Tree is a type of CLOS network architecture.<\/p>\n\n\n\n<p>Compared to the traditional tree structure, a Fat-Tree is more like a real tree, with thicker branches near the root. From the leaves to the root, the network bandwidth does not converge.<\/p>\n\n\n\n<p>The basic idea: use a large number of low-performance switches to build a large-scale non-blocking network. For any communication pattern, there is always a path that allows them to achieve the bandwidth of the network card.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Fat-Tree-architecture.png\" alt=\"Fat-Tree architecture\" class=\"wp-image-7110\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Fat-Tree-architecture.png 821w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Fat-Tree-architecture-300x129.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Fat-Tree-architecture-768x330.png 768w\" sizes=\"(max-width: 821px) 100vw, 821px\" \/><\/figure>\n\n\n\n<p>After the Fat-Tree architecture was introduced to the data center, the data center became a traditional three-layer structure:<\/p>\n\n\n\n<p><strong>Access layer<\/strong>: used to connect all the computing nodes. Usually in the form of a rack switch (TOR, Top of Rack).<\/p>\n\n\n\n<p><strong>Aggregation layer<\/strong>: used for interconnection of the access layer, and as the boundary of the second and third layers of the aggregation area. Various services such as firewalls, load balancing, etc. are also deployed here.<\/p>\n\n\n\n<p><strong>Core layer<\/strong>: used for interconnection of the aggregation layer, and to implement the third-layer communication between the entire data center and the external network.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/three-layer-structure.png\" alt=\"three-layer structure\" class=\"wp-image-7111\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/three-layer-structure.png 697w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/three-layer-structure-300x151.png 300w\" sizes=\"(max-width: 697px) 100vw, 697px\" \/><\/figure>\n\n\n\n<p>The disadvantages of the Fat-Tree architecture:<\/p>\n\n\n\n<p><strong>Resource waste<\/strong>: In the traditional three-layer structure, a lower-layer switch will be connected to two upper-layer switches through two links. Because the STP protocol (Spanning Tree Protocol) is used, only one link actually carries the traffic. The other uplink is blocked (only used for backup). This causes bandwidth waste.<\/p>\n\n\n\n<p><strong>Large fault domain<\/strong>: The STP protocol, due to its own algorithm, needs to reconverge when the network topology changes, which can easily cause faults and affect the network of the entire VLAN.<\/p>\n\n\n\n<p><strong>Not suitable for east-west traffic<\/strong>: The communication between servers and servers requires passing through the access switch, the aggregation switch, and the core switch.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Not-suitable-for-east-west-traffic.png\" alt=\"Not suitable for east-west traffic\" class=\"wp-image-7112\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Not-suitable-for-east-west-traffic.png 937w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Not-suitable-for-east-west-traffic-300x132.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Not-suitable-for-east-west-traffic-768x337.png 768w\" sizes=\"(max-width: 937px) 100vw, 937px\" \/><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"h-spine-leaf-network\"><span class=\"ez-toc-section\" id=\"Spine-Leaf_network\"><\/span><strong>Spine-Leaf network<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Like the Fat-Tree structure, it belongs to the CLOS network model.<\/p>\n\n\n\n<p>Compared to the traditional three-layer network architecture, the Spine-Leaf network has been flattened and turned into a two-layer architecture.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Spine-Leaf-network.png\" alt=\"Spine-Leaf network\" class=\"wp-image-7113\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Spine-Leaf-network.png 629w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/Spine-Leaf-network-300x151.png 300w\" sizes=\"(max-width: 629px) 100vw, 629px\" \/><\/figure>\n\n\n\n<p>Leaf switch, equivalent to the access switch in the traditional three-layer architecture, as TOR (Top Of Rack) directly connected to the physical server. Above the leaf switch is the third layer network, each is an independent L2 broadcast domain. If the servers under two leaf switches need to communicate, they need to be forwarded by the spine switch.<\/p>\n\n\n\n<p>Spine switch, equivalent to the core switch. The leaf and spine switches dynamically select multiple paths through ECMP (Equal Cost Multi Path).<\/p>\n\n\n\n<p>The number of downlink ports of the spine switch determines the number of leaf switches. The number of uplink ports of the leaf switch determines the number of spine switches. They jointly determine the scale of the Spine-Leaf network.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/spine-leaf-topology.png\" alt=\"spine-leaf topology\" class=\"wp-image-7114\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/spine-leaf-topology.png 1013w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/spine-leaf-topology-300x110.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/spine-leaf-topology-768x283.png 768w\" sizes=\"(max-width: 1013px) 100vw, 1013px\" \/><\/figure>\n\n\n\n<p><strong>The Advantages of Spine-Leaf Network<\/strong><\/p>\n\n\n\n<p>High bandwidth utilization<\/p>\n\n\n\n<p>Each leaf switch\u2019s uplink works in a load-balancing manner, making full use of the bandwidth.<\/p>\n\n\n\n<p>Predictable network latency<\/p>\n\n\n\n<p>In the above model, the number of communication paths between leaf switches can be determined, and only one spine switch is required for each path. The east-west network latency is predictable.<\/p>\n\n\n\n<p>Good scalability<\/p>\n\n\n\n<p>When the bandwidth is insufficient, the number of spine switches can be increased to scale the bandwidth horizontally. When the number of servers increases, the number of spine switches can also be increased to expand the data center scale. Planning and expansion are very convenient.<\/p>\n\n\n\n<p>Reduced requirements for switches<\/p>\n\n\n\n<p>North-south traffic can go out from the leaf nodes or the spine nodes. East-west traffic is distributed on multiple paths. Expensive high-performance high-bandwidth switches are not required.<\/p>\n\n\n\n<p>High security and availability<\/p>\n\n\n\n<p>Traditional networks use STP protocol, which will reconverge when a device fails, affecting network performance or even causing faults. In the Spine-Leaf architecture, when a device fails, there is no need to reconverge, and the traffic continues to pass through other normal paths. The network connectivity is not affected, and the bandwidth is only reduced by the bandwidth of one path. The performance impact is negligible.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"h-infiniband\"><span class=\"ez-toc-section\" id=\"InfiniBand\"><\/span><strong>InfiniBand<\/strong><strong><\/strong><span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"h-rdma-remote-direct-memory-access-protocol\"><span class=\"ez-toc-section\" id=\"RDMA_Remote_Direct_Memory_Access_protocol\"><\/span><strong>RDMA (Remote Direct Memory Access) protocol<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>In the traditional TCP\/IP, the data from the network card is first copied to the kernel memory, and then copied to the application storage space, or the data is copied from the application space to the kernel memory and then sent to the Internet via the network card. This I\/O operation mode requires the conversion of the kernel memory. It increases the length of the data flow transmission path, increases the CPU load, and also increases the transmission latency.<\/p>\n\n\n\n<p>RDMA\u2019s kernel bypass mechanism allows direct data read and write between the application and the network card, reducing the data transmission latency within the server to close to 1us.<\/p>\n\n\n\n<p>At the same time, RDMA\u2019s memory zero-copy mechanism allows the receiver to directly read data from the sender\u2019s memory, bypassing the participation of the kernel memory, greatly reducing the CPU load and improving the CPU efficiency.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RDMA.png\" alt=\"RDMA\" class=\"wp-image-7115\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RDMA.png 678w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RDMA-300x136.png 300w\" sizes=\"(max-width: 678px) 100vw, 678px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image aligncenter size-large is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-vs-rdma-1024x520.png\" alt=\"infiniband vs rdma\" class=\"wp-image-7116\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-vs-rdma-1024x520.png 1024w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-vs-rdma-300x152.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-vs-rdma-768x390.png 768w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-vs-rdma.png 1066w\" sizes=\"(max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"h-the-background-of-infiniband\"><span class=\"ez-toc-section\" id=\"The_background_of_InfiniBand\"><\/span><strong>The background of InfiniBand<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>InfiniBand (abbreviated as IB) is a powerful communication technology protocol. Its English translation is \u201cinfinite bandwidth\u201d. It was born in the 1990s, to replace the PCI (Peripheral Component Interconnect) bus. The PCI bus was introduced by Intel into the PC architecture, and the upgrade speed was slow, which greatly limited the I\/O performance and became the bottleneck of the entire system.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/The-background-of-InfiniBand.png\" alt=\"The background of InfiniBand\" class=\"wp-image-7117\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/The-background-of-InfiniBand.png 571w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/The-background-of-InfiniBand-300x181.png 300w\" sizes=\"(max-width: 571px) 100vw, 571px\" \/><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"h-the-development-history-of-infiniband\"><span class=\"ez-toc-section\" id=\"The_development_history_of_InfiniBand\"><\/span><strong>The development history of InfiniBand<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>In the 1990s, Intel, Microsoft, and SUN led the development of the \u201cNext Generation I\/O (NGIO)\u201d technology standard, while IBM, Compaq, and HP led the development of \u201cFuture I\/O (FIO)\u201d.<\/p>\n\n\n\n<p>In 1999, the FIO Developers Forum and the NGIO Forum merged and established the <a href=\"https:\/\/www.fibermall.com\/collection\/infiniband.htm\" target=\"_blank\" rel=\"noreferrer noopener\">InfiniBand<\/a> Trade Association (IBTA).<\/p>\n\n\n\n<p>In 2000, the InfiniBand architecture specification version 1.0 was officially released.<\/p>\n\n\n\n<p>In May 1999, several employees who left Intel and Galileo Technology founded a chip company in Israel and named it Mellanox.<\/p>\n\n\n\n<p>After Mellanox was established, it joined NGIO. Later, Mellanox joined the InfiniBand camp. In 2001, they launched their first InfiniBand product. Starting in<\/p>\n\n\n\n<p>2003, InfiniBand turned to a new application field, which is computer cluster interconnection.<\/p>\n\n\n\n<p>In 2004, another important InfiniBand non-profit organization was born-OFA (Open Fabrics Alliance).<\/p>\n\n\n\n<p>In 2005, InfiniBand found another new scenario-the connection of storage devices.<\/p>\n\n\n\n<p>Since then, InfiniBand has entered a stage of rapid development.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/The-development-history-of-InfiniBand.png\" alt=\"The development history of InfiniBand\" class=\"wp-image-7118\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/The-development-history-of-InfiniBand.png 800w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/The-development-history-of-InfiniBand-300x113.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/The-development-history-of-InfiniBand-768x289.png 768w\" sizes=\"(max-width: 800px) 100vw, 800px\" \/><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"h-infiniband-network-architecture\"><span class=\"ez-toc-section\" id=\"InfiniBand_Network_Architecture\"><\/span><strong>InfiniBand Network Architecture<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>InfiniBand is a channel-based structure, consisting of four main components:<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>HCA (Host Channel Adapter), which connects the host to the InfiniBand network.<\/li>\n\n\n\n<li>TCA (Target Channel Adapter), which connects the target device (such as storage) to the InfiniBand network.<\/li>\n\n\n\n<li>InfiniBand link, which can be a cable, fiber, or on-board link, connects the channel adapters to the switches or routers.<\/li>\n\n\n\n<li>InfiniBand switch and router, which provide network connectivity and routing for the InfiniBand network.<\/li>\n\n\n\n<li>Channel adapters are used to establish InfiniBand channels. All transmissions start or end with channel adapters, to ensure security or work at a given QoS (Quality of Service) level.<\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/InfiniBand-Network-Architecture.png\" alt=\"InfiniBand Network Architecture\" class=\"wp-image-7119\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/InfiniBand-Network-Architecture.png 896w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/InfiniBand-Network-Architecture-300x171.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/InfiniBand-Network-Architecture-768x439.png 768w\" sizes=\"(max-width: 896px) 100vw, 896px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/subnet.png\" alt=\"subnet\" class=\"wp-image-7120\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/subnet.png 980w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/subnet-300x172.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/subnet-768x440.png 768w\" sizes=\"(max-width: 980px) 100vw, 980px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-layer.png\" alt=\"infiniband layer\" class=\"wp-image-7121\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-layer.png 894w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-layer-300x178.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-layer-768x455.png 768w\" sizes=\"(max-width: 894px) 100vw, 894px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-message.png\" alt=\"infiniband message\" class=\"wp-image-7122\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-message.png 897w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-message-300x166.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-message-768x424.png 768w\" sizes=\"(max-width: 897px) 100vw, 897px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-transmission.png\" alt=\"infiniband transmission\" class=\"wp-image-7123\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-transmission.png 838w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-transmission-300x178.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-transmission-768x456.png 768w\" sizes=\"(max-width: 838px) 100vw, 838px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/interface-standard.png\" alt=\"interface standard\" class=\"wp-image-7124\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/interface-standard.png 792w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/interface-standard-300x162.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/interface-standard-768x415.png 768w\" sizes=\"(max-width: 792px) 100vw, 792px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/interface-rate.png\" alt=\"interface rate\" class=\"wp-image-7125\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/interface-rate.png 946w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/interface-rate-300x142.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/interface-rate-768x363.png 768w\" sizes=\"(max-width: 946px) 100vw, 946px\" \/><\/figure>\n\n\n\n<p>Mellanox, acquired by Nvidia in 2020. Since then, it has been widely used in AI large model training.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-1.png\" alt=\"infiniband\" class=\"wp-image-7126\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-1.png 782w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-1-300x138.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/infiniband-1-768x353.png 768w\" sizes=\"(max-width: 782px) 100vw, 782px\" \/><\/figure>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"h-roce\"><span class=\"ez-toc-section\" id=\"RoCE\"><\/span><strong>RoCE <\/strong><span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"h-the-birth-of-roce\"><span class=\"ez-toc-section\" id=\"The_birth_of_RoCE\"><\/span><strong>The birth of RoCE<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>In April 2010, IBTA released RoCE (RDMA over Converged Ethernet), which \u201cported\u201d&nbsp;the RDMA technology in InfiniBand to Ethernet. In 2014, they proposed a more mature RoCEv2. With RoCEv2, Ethernet greatly narrowed the technical performance gap with InfiniBand, and combined with its inherent cost and compatibility advantages, it started to fight back.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE.png\" alt=\"RoCE\" class=\"wp-image-7127\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE.png 646w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-300x101.png 300w\" sizes=\"(max-width: 646px) 100vw, 646px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-embed is-type-video is-provider-youtube wp-block-embed-youtube wp-embed-aspect-16-9 wp-has-aspect-ratio\"><div class=\"wp-block-embed__wrapper\">\n<div class=\"ast-oembed-container \" style=\"height: 100%;\"><iframe title=\"InfiniBand and RoCE: Artificial Intelligence Data Centers | FiberMall\" width=\"500\" height=\"281\" src=\"https:\/\/www.youtube.com\/embed\/eGoP2wPoaEM?feature=oembed\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen><\/iframe><\/div>\n<\/div><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"h-roce-v2\"><span class=\"ez-toc-section\" id=\"RoCE_V2\"><\/span><strong>RoCE V2<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>RoCE v1: An RDMA protocol based on the Ethernet link layer (the switch needs to support flow control technologies such as PFC, to ensure reliable transmission at the physical layer), which allows communication between two hosts in the same VLAN. RoCE V2: Overcomes the limitation of RoCE v1 being bound to a single VLAN. By changing the packet encapsulation, including IP and UDP headers, RoCE 2 can now be used across L2 and L3 networks.<\/p>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/roce-working-principle.png\" alt=\"roce working principle\" class=\"wp-image-7128\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/roce-working-principle.png 812w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/roce-working-principle-300x133.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/roce-working-principle-768x340.png 768w\" sizes=\"(max-width: 812px) 100vw, 812px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/roce-message-structure.png\" alt=\"roce message structure\" class=\"wp-image-7129\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/roce-message-structure.png 770w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/roce-message-structure-300x146.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/roce-message-structure-768x375.png 768w\" sizes=\"(max-width: 770px) 100vw, 770px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img decoding=\"async\" src=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/IB-and-roce.png\" alt=\"IB and roce\" class=\"wp-image-7130\" style=\"width:800px\" width=\"800\" srcset=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/IB-and-roce.png 777w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/IB-and-roce-300x185.png 300w, https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/IB-and-roce-768x472.png 768w\" sizes=\"(max-width: 777px) 100vw, 777px\" \/><\/figure>\n<style>\r\n\r\n        .lwrp.link-whisper-related-posts{\r\n            \r\n            margin-top: 40px;\nmargin-bottom: 30px;\r\n        }\r\n        .lwrp .lwrp-title{\r\n            \r\n            \r\n        }\r\n        .lwrp .lwrp-description{\r\n            \r\n            \r\n\r\n        }\r\n        .lwrp .lwrp-list-container{\r\n        }\r\n        .lwrp .lwrp-list-multi-container{\r\n            display: flex;\r\n        }\r\n        .lwrp .lwrp-list-double{\r\n            width: 48%;\r\n        }\r\n        .lwrp .lwrp-list-triple{\r\n            width: 32%;\r\n        }\r\n        .lwrp .lwrp-list-row-container{\r\n            display: flex;\r\n            justify-content: space-between;\r\n        }\r\n        .lwrp .lwrp-list-row-container .lwrp-list-item{\r\n            width: calc(100% - 20px);\r\n        }\r\n        .lwrp .lwrp-list-item:not(.lwrp-no-posts-message-item){\r\n            \r\n            list-style: decimal;\r\n        }\r\n        .lwrp .lwrp-list-item img{\r\n            max-width: 100%;\r\n            height: auto;\r\n        }\r\n        .lwrp .lwrp-list-item.lwrp-empty-list-item{\r\n            background: initial !important;\r\n        }\r\n        .lwrp .lwrp-list-item .lwrp-list-link .lwrp-list-link-title-text,\r\n        .lwrp .lwrp-list-item .lwrp-list-no-posts-message{\r\n            \r\n                \r\n        }\r\n        @media screen and (max-width: 480px) {\r\n            .lwrp.link-whisper-related-posts{\r\n                \r\n                \r\n            }\r\n            .lwrp .lwrp-title{\r\n                \r\n                \r\n            }\r\n            .lwrp .lwrp-description{\r\n                \r\n                \r\n            }\r\n            .lwrp .lwrp-list-multi-container{\r\n                flex-direction: column;\r\n            }\r\n            .lwrp .lwrp-list-multi-container ul.lwrp-list{\r\n                margin-top: 0px;\r\n                margin-bottom: 0px;\r\n                padding-top: 0px;\r\n                padding-bottom: 0px;\r\n            }\r\n            .lwrp .lwrp-list-double,\r\n            .lwrp .lwrp-list-triple{\r\n                width: 100%;\r\n            }\r\n            .lwrp .lwrp-list-row-container{\r\n                justify-content: initial;\r\n                flex-direction: column;\r\n            }\r\n            .lwrp .lwrp-list-row-container .lwrp-list-item{\r\n                width: 100%;\r\n            }\r\n            .lwrp .lwrp-list-item:not(.lwrp-no-posts-message-item){\r\n                \r\n                \r\n            }\r\n            .lwrp .lwrp-list-item .lwrp-list-link .lwrp-list-link-title-text,\r\n            .lwrp .lwrp-list-item .lwrp-list-no-posts-message{\r\n                \r\n                    \r\n            }\r\n        }<\/style>\r\n<div id=\"link-whisper-related-posts-widget\" class=\"link-whisper-related-posts lwrp\">\r\n            <h3 class=\"lwrp-title\">Related Posts<\/h3>    \r\n        <div class=\"lwrp-list-container\">\r\n                                            <ul class=\"lwrp-list lwrp-list-single\">\r\n                    <li class=\"lwrp-list-item\"><a href=\"https:\/\/www.fibermall.com\/blog\/infiniband-adapter.htm\" class=\"lwrp-list-link\"><span class=\"lwrp-list-link-title-text\">Top-Performing Infiniband Adapter Solutions for High-Speed Data Transfers: A Comprehensive Guide<\/span><\/a><\/li><li class=\"lwrp-list-item\"><a href=\"https:\/\/www.fibermall.com\/blog\/nvidia-nvlink-and-nvswitch-evolution.htm\" class=\"lwrp-list-link\"><span class=\"lwrp-list-link-title-text\">Understanding Nvidia&#8217;s NvLink and NvSwitch Evolution: Topology and Rates<\/span><\/a><\/li><li class=\"lwrp-list-item\"><a href=\"https:\/\/www.fibermall.com\/blog\/nvidia-dgx-gh200.htm\" class=\"lwrp-list-link\"><span class=\"lwrp-list-link-title-text\">The Revolutionary Nvidia DGX GH200: Powering the Future of AI Supercomputers<\/span><\/a><\/li><li class=\"lwrp-list-item\"><a href=\"https:\/\/www.fibermall.com\/blog\/sfp28-switch.htm\" class=\"lwrp-list-link\"><span class=\"lwrp-list-link-title-text\">Everything You Need to Know About SFP28 Switches: 25G SFP28 Managed Switch for Pro AV and Cisco Networks<\/span><\/a><\/li><li class=\"lwrp-list-item\"><a href=\"https:\/\/www.fibermall.com\/blog\/25g-sfp28-cwdm.htm\" class=\"lwrp-list-link\"><span class=\"lwrp-list-link-title-text\">SFP28 25G CWDM Transceiver Module: The Future of Ethernet Connectivity<\/span><\/a><\/li>                <\/ul>\r\n                        <\/div>\r\n<\/div>","protected":false},"excerpt":{"rendered":"<p>Data Center Network Architecture Crossbar architecture Clos architecture Fat-Tree architecture A Fat-Tree is a type of CLOS network architecture. Compared to the traditional tree structure, a Fat-Tree is more like a real tree, with thicker branches near the root. From the leaves to the root, the network bandwidth does not converge. The basic idea: use [&hellip;]<\/p>\n","protected":false},"author":4,"featured_media":7135,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_acf_changed":false,"site-sidebar-layout":"default","site-content-layout":"","ast-site-content-layout":"default","site-content-style":"default","site-sidebar-style":"default","ast-global-header-display":"","ast-banner-title-visibility":"","ast-main-header-display":"","ast-hfb-above-header-display":"","ast-hfb-below-header-display":"","ast-hfb-mobile-header-display":"","site-post-title":"","ast-breadcrumbs-content":"","ast-featured-img":"","footer-sml-layout":"","theme-transparent-header-meta":"","adv-header-id-meta":"","stick-header-meta":"","header-above-stick-meta":"","header-main-stick-meta":"","header-below-stick-meta":"","astra-migrate-meta-layouts":"set","ast-page-background-enabled":"default","ast-page-background-meta":{"desktop":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"ast-content-background-meta":{"desktop":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"footnotes":"","_wpscppro_dont_share_socialmedia":false,"_wpscppro_custom_social_share_image":0,"_facebook_share_type":"","_twitter_share_type":"","_linkedin_share_type":"","_pinterest_share_type":"","_linkedin_share_type_page":"","_instagram_share_type":"","_medium_share_type":"","_threads_share_type":"","_google_business_share_type":"","_selected_social_profile":[],"_wpsp_enable_custom_social_template":false,"_wpsp_social_scheduling":{"enabled":false,"datetime":null,"platforms":[],"status":"template_only","dateOption":"today","timeOption":"now","customDays":"","customHours":"","customDate":"","customTime":"","schedulingType":"absolute"},"_wpsp_active_default_template":true},"categories":[2,29],"tags":[],"class_list":["post-7106","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-blog","category-networking"],"acf":[],"yoast_head":"<!-- This site is optimized with the Yoast SEO Premium plugin v20.13 (Yoast SEO v25.8) - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>What is the Difference Between InfiniBand and RoCE? - fibermall.com<\/title>\n<meta name=\"description\" content=\"InfiniBand, RoCEv1, RoCEv2, and iWARP are the four implementations of RDMA technology. The prevalent network solutions are InfiniBand and RoCEv2.\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"What is the Difference Between InfiniBand and RoCE?\" \/>\n<meta property=\"og:description\" content=\"Data Center Network Architecture Crossbar architecture A type of architecture derived from the earliest telephone switching network (crossbar switch)\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\" \/>\n<meta property=\"og:site_name\" content=\"fibermall.com\" \/>\n<meta property=\"article:published_time\" content=\"2024-01-19T09:57:51+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-03-15T10:24:34+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-1.png\" \/>\n\t<meta property=\"og:image:width\" content=\"600\" \/>\n\t<meta property=\"og:image:height\" content=\"400\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/png\" \/>\n<meta name=\"author\" content=\"Mia\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"Written by\" \/>\n\t<meta name=\"twitter:data1\" content=\"Mia\" \/>\n\t<meta name=\"twitter:label2\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data2\" content=\"11 minutes\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#article\",\"isPartOf\":{\"@id\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\"},\"author\":{\"name\":\"Mia\",\"@id\":\"https:\/\/www.fibermall.com\/blog.htm\/#\/schema\/person\/56a37c7b0e1e8c61823cd6ef48e8dce3\"},\"headline\":\"What is the Difference Between InfiniBand and RoCE?\",\"datePublished\":\"2024-01-19T09:57:51+00:00\",\"dateModified\":\"2024-03-15T10:24:34+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\"},\"wordCount\":1373,\"commentCount\":0,\"publisher\":{\"@id\":\"https:\/\/www.fibermall.com\/blog.htm\/#organization\"},\"image\":{\"@id\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#primaryimage\"},\"thumbnailUrl\":\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-1.png\",\"articleSection\":[\"Blog\",\"Networking\"],\"inLanguage\":\"en-US\"},{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\",\"url\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\",\"name\":\"What is the Difference Between InfiniBand and RoCE? - fibermall.com\",\"isPartOf\":{\"@id\":\"https:\/\/www.fibermall.com\/blog.htm\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#primaryimage\"},\"image\":{\"@id\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#primaryimage\"},\"thumbnailUrl\":\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-1.png\",\"datePublished\":\"2024-01-19T09:57:51+00:00\",\"dateModified\":\"2024-03-15T10:24:34+00:00\",\"description\":\"InfiniBand, RoCEv1, RoCEv2, and iWARP are the four implementations of RDMA technology. The prevalent network solutions are InfiniBand and RoCEv2.\",\"breadcrumb\":{\"@id\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#primaryimage\",\"url\":\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-1.png\",\"contentUrl\":\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-1.png\",\"width\":600,\"height\":400,\"caption\":\"infiniband vs RoCE\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\/\/www.fibermall.com\/blog.htm\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"What is the Difference Between InfiniBand and RoCE?\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.fibermall.com\/blog.htm\/#website\",\"url\":\"https:\/\/www.fibermall.com\/blog.htm\/\",\"name\":\"fibermall.com\",\"description\":\"Optical Communication Expert\",\"publisher\":{\"@id\":\"https:\/\/www.fibermall.com\/blog.htm\/#organization\"},\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.fibermall.com\/blog.htm\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":\"Organization\",\"@id\":\"https:\/\/www.fibermall.com\/blog.htm\/#organization\",\"name\":\"fibermall.com\",\"url\":\"https:\/\/www.fibermall.com\/blog.htm\/\",\"logo\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\/\/www.fibermall.com\/blog.htm\/#\/schema\/logo\/image\/\",\"url\":\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/12\/cropped-Fiber-Mall-Logo.jpg\",\"contentUrl\":\"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/12\/cropped-Fiber-Mall-Logo.jpg\",\"width\":200,\"height\":67,\"caption\":\"fibermall.com\"},\"image\":{\"@id\":\"https:\/\/www.fibermall.com\/blog.htm\/#\/schema\/logo\/image\/\"}},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.fibermall.com\/blog.htm\/#\/schema\/person\/56a37c7b0e1e8c61823cd6ef48e8dce3\",\"name\":\"Mia\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\/\/www.fibermall.com\/blog.htm\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/2e83831e57346d5d451c45e43aa96d14fc758d8f72f7304810e123b36585382b?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/2e83831e57346d5d451c45e43aa96d14fc758d8f72f7304810e123b36585382b?s=96&d=mm&r=g\",\"caption\":\"Mia\"},\"description\":\"Optical Transmission Consultant\",\"sameAs\":[\"https:\/\/www.fibermall.com\/\"],\"url\":\"https:\/\/www.fibermall.com\/blog.htm?author=4\"}]}<\/script>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"What is the Difference Between InfiniBand and RoCE? - fibermall.com","description":"InfiniBand, RoCEv1, RoCEv2, and iWARP are the four implementations of RDMA technology. The prevalent network solutions are InfiniBand and RoCEv2.","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm","og_locale":"en_US","og_type":"article","og_title":"What is the Difference Between InfiniBand and RoCE?","og_description":"Data Center Network Architecture Crossbar architecture A type of architecture derived from the earliest telephone switching network (crossbar switch)","og_url":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm","og_site_name":"fibermall.com","article_published_time":"2024-01-19T09:57:51+00:00","article_modified_time":"2024-03-15T10:24:34+00:00","og_image":[{"width":600,"height":400,"url":"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-1.png","type":"image\/png"}],"author":"Mia","twitter_card":"summary_large_image","twitter_misc":{"Written by":"Mia","Est. reading time":"11 minutes"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#article","isPartOf":{"@id":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm"},"author":{"name":"Mia","@id":"https:\/\/www.fibermall.com\/blog.htm\/#\/schema\/person\/56a37c7b0e1e8c61823cd6ef48e8dce3"},"headline":"What is the Difference Between InfiniBand and RoCE?","datePublished":"2024-01-19T09:57:51+00:00","dateModified":"2024-03-15T10:24:34+00:00","mainEntityOfPage":{"@id":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm"},"wordCount":1373,"commentCount":0,"publisher":{"@id":"https:\/\/www.fibermall.com\/blog.htm\/#organization"},"image":{"@id":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#primaryimage"},"thumbnailUrl":"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-1.png","articleSection":["Blog","Networking"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm","url":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm","name":"What is the Difference Between InfiniBand and RoCE? - fibermall.com","isPartOf":{"@id":"https:\/\/www.fibermall.com\/blog.htm\/#website"},"primaryImageOfPage":{"@id":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#primaryimage"},"image":{"@id":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#primaryimage"},"thumbnailUrl":"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-1.png","datePublished":"2024-01-19T09:57:51+00:00","dateModified":"2024-03-15T10:24:34+00:00","description":"InfiniBand, RoCEv1, RoCEv2, and iWARP are the four implementations of RDMA technology. The prevalent network solutions are InfiniBand and RoCEv2.","breadcrumb":{"@id":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#primaryimage","url":"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-1.png","contentUrl":"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/01\/RoCE-1.png","width":600,"height":400,"caption":"infiniband vs RoCE"},{"@type":"BreadcrumbList","@id":"https:\/\/www.fibermall.com\/blog\/what-is-the-difference-between-infiniband-and-roce.htm#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/www.fibermall.com\/blog.htm"},{"@type":"ListItem","position":2,"name":"What is the Difference Between InfiniBand and RoCE?"}]},{"@type":"WebSite","@id":"https:\/\/www.fibermall.com\/blog.htm\/#website","url":"https:\/\/www.fibermall.com\/blog.htm\/","name":"fibermall.com","description":"Optical Communication Expert","publisher":{"@id":"https:\/\/www.fibermall.com\/blog.htm\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.fibermall.com\/blog.htm\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/www.fibermall.com\/blog.htm\/#organization","name":"fibermall.com","url":"https:\/\/www.fibermall.com\/blog.htm\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.fibermall.com\/blog.htm\/#\/schema\/logo\/image\/","url":"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/12\/cropped-Fiber-Mall-Logo.jpg","contentUrl":"https:\/\/www.fibermall.com\/blog\/wp-content\/uploads\/2024\/12\/cropped-Fiber-Mall-Logo.jpg","width":200,"height":67,"caption":"fibermall.com"},"image":{"@id":"https:\/\/www.fibermall.com\/blog.htm\/#\/schema\/logo\/image\/"}},{"@type":"Person","@id":"https:\/\/www.fibermall.com\/blog.htm\/#\/schema\/person\/56a37c7b0e1e8c61823cd6ef48e8dce3","name":"Mia","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.fibermall.com\/blog.htm\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/2e83831e57346d5d451c45e43aa96d14fc758d8f72f7304810e123b36585382b?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/2e83831e57346d5d451c45e43aa96d14fc758d8f72f7304810e123b36585382b?s=96&d=mm&r=g","caption":"Mia"},"description":"Optical Transmission Consultant","sameAs":["https:\/\/www.fibermall.com\/"],"url":"https:\/\/www.fibermall.com\/blog.htm?author=4"}]}},"_links":{"self":[{"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=\/wp\/v2\/posts\/7106","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=\/wp\/v2\/users\/4"}],"replies":[{"embeddable":true,"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=7106"}],"version-history":[{"count":7,"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=\/wp\/v2\/posts\/7106\/revisions"}],"predecessor-version":[{"id":7427,"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=\/wp\/v2\/posts\/7106\/revisions\/7427"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=\/wp\/v2\/media\/7135"}],"wp:attachment":[{"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=7106"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=7106"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.fibermall.com\/blog.htm\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=7106"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}