Skip to content

Commit 24241b1

Browse files
author
Gregory Price
committed
Linux: driver - example configuration intra-hb interleave
1 parent 312f437 commit 24241b1

File tree

1 file changed

+287
-2
lines changed

1 file changed

+287
-2
lines changed

‎doc/linux/example-configurations/intra-hb-interleave.rst‎

Lines changed: 287 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,291 @@
1-
.. 2 devices on 1 host bridge
1+
.. intra-host-bridge interleave
22
33
============================
44
Intra-Host-Bridge Interleave
55
============================
6-
todo
6+
This cxl-cli configuration dump shows the following host configuration:
7+
8+
* A single socket system with one CXL root
9+
* CXL Root has Four (4) CXL Host Bridges
10+
* One (1) CXL Host Bridges has two CXL Memory Expanders Attached
11+
* The Host bridge decoder is programmed to interleave across the expanders.
12+
13+
This output is generated by :code:`cxl list -v` and describes the relationships
14+
between objects exposed in :code:`/sys/bus/cxl/devices/`.
15+
16+
::
17+
18+
[
19+
{
20+
"bus":"root0",
21+
"provider":"ACPI.CXL",
22+
"nr_dports":4,
23+
"dports":[
24+
{
25+
"dport":"pci0000:00",
26+
"alias":"ACPI0016:01",
27+
"id":0
28+
},
29+
{
30+
"dport":"pci0000:a8",
31+
"alias":"ACPI0016:02",
32+
"id":4
33+
},
34+
{
35+
"dport":"pci0000:2a",
36+
"alias":"ACPI0016:03",
37+
"id":1
38+
},
39+
{
40+
"dport":"pci0000:d2",
41+
"alias":"ACPI0016:00",
42+
"id":5
43+
}
44+
],
45+
46+
This chunk shows the CXL "bus" (root0) has 4 downstream ports attached to CXL
47+
Host Bridges. The `Root` can be considered the singular upstream port attached
48+
to the platform's memory controller - which routes memory requests to it.
49+
50+
The `ports:root0` section lays out how each of these downstream ports are
51+
configured. If a port is not configured (id's 0 and 1), they are omitted.
52+
53+
::
54+
55+
"ports:root0":[
56+
{
57+
"port":"port1",
58+
"host":"pci0000:d2",
59+
"depth":1,
60+
"nr_dports":3,
61+
"dports":[
62+
{
63+
"dport":"0000:d2:01.1",
64+
"alias":"device:02",
65+
"id":0
66+
},
67+
{
68+
"dport":"0000:d2:01.3",
69+
"alias":"device:05",
70+
"id":2
71+
},
72+
{
73+
"dport":"0000:d2:07.1",
74+
"alias":"device:0d",
75+
"id":113
76+
}
77+
],
78+
79+
This chunk shows the available downstream ports associated with the CXL Host
80+
Bridge :code:`port1`. In this case, :code:`port1` has 3 avaiable downstream
81+
ports: :code:`dport1`, :code:`dport2`, and :code:`dport113`..
82+
83+
::
84+
85+
"endpoints:port1":[
86+
{
87+
"endpoint":"endpoint5",
88+
"host":"mem0",
89+
"parent_dport":"0000:d2:01.1",
90+
"depth":2,
91+
"memdev":{
92+
"memdev":"mem0",
93+
"ram_size":137438953472,
94+
"serial":0,
95+
"numa_node":0,
96+
"host":"0000:d3:00.0"
97+
},
98+
"decoders:endpoint5":[
99+
{
100+
"decoder":"decoder5.0",
101+
"resource":825975898112,
102+
"size":274877906944,
103+
"interleave_ways":2,
104+
"interleave_granularity":256,
105+
"region":"region0",
106+
"dpa_resource":0,
107+
"dpa_size":274877906944,
108+
"mode":"ram"
109+
}
110+
]
111+
},
112+
{
113+
"endpoint":"endpoint6",
114+
"host":"mem1",
115+
"parent_dport":"0000:d2:01.3,
116+
"depth":2,
117+
"memdev":{
118+
"memdev":"mem1",
119+
"ram_size":137438953472,
120+
"serial":0,
121+
"numa_node":0,
122+
"host":"0000:a9:00.0"
123+
},
124+
"decoders:endpoint6":[
125+
{
126+
"decoder":"decoder6.0",
127+
"resource":825975898112,
128+
"size":274877906944,
129+
"interleave_ways":2,
130+
"interleave_granularity":256,
131+
"region":"region0",
132+
"dpa_resource":0,
133+
"dpa_size":274877906944,
134+
"mode":"ram"
135+
}
136+
]
137+
}
138+
],
139+
140+
This chunk shows the endpoints attached to the host bridge :code:`port1`.
141+
142+
:code:`endpoint5` contains a single configured decoder :code:`decoder5.0`
143+
which has the same interleave configuration memory region they belong to
144+
(show later).
145+
146+
Next we have the decoders belonging to the host bridge:
147+
148+
::
149+
150+
"decoders:port1":[
151+
{
152+
"decoder":"decoder1.0",
153+
"resource":825975898112,
154+
"size":274877906944,
155+
"interleave_ways":2,
156+
"interleave_granularity":256,
157+
"region":"region0",
158+
"nr_targets":2,
159+
"targets":[
160+
{
161+
"target":"0000:d2:01.1",
162+
"alias":"device:02",
163+
"position":0,
164+
"id":0
165+
},
166+
{
167+
"target":"0000:d2:01.3",
168+
"alias":"device:05",
169+
"position":1,
170+
"id":0
171+
}
172+
]
173+
}
174+
]
175+
},
176+
177+
Host Bridge :code:`port1` has a single decoder (:code:`decoder1.0`) with two
178+
targets: :code:`dport1` and :code:`dport3` - which are attached to
179+
:code:`endpoint5` and :code:`endpoint6` respectively.
180+
181+
The host bridge decoder interleaves these devices at a 256 byte granularity.
182+
183+
The next chunk shows the three CXL host bridges without attached endpoints.
184+
185+
::
186+
187+
{
188+
"port":"port2",
189+
"host":"pci0000:00",
190+
"depth":1,
191+
"nr_dports":2,
192+
"dports":[
193+
{
194+
"dport":"0000:00:01.3",
195+
"alias":"device:55",
196+
"id":2
197+
},
198+
{
199+
"dport":"0000:00:07.1",
200+
"alias":"device:5d",
201+
"id":113
202+
}
203+
]
204+
},
205+
{
206+
"port":"port3",
207+
"host":"pci0000:a8",
208+
"depth":1,
209+
"nr_dports":1,
210+
"dports":[
211+
{
212+
"dport":"0000:a8:01.1",
213+
"alias":"device:c3",
214+
"id":0
215+
}
216+
],
217+
},
218+
{
219+
"port":"port4",
220+
"host":"pci0000:2a",
221+
"depth":1,
222+
"nr_dports":1,
223+
"dports":[
224+
{
225+
"dport":"0000:2a:01.1",
226+
"alias":"device:d0",
227+
"id":0
228+
}
229+
]
230+
}
231+
],
232+
233+
Next we have the `Root Decoders` belonging to :code:`root0`. This root decoder
234+
applies the interleave across the downstream ports :code:`port1` and
235+
:code:`port3` - with a granularity of 256 bytes.
236+
237+
This information is generated by the CXL driver reading the ACPI CEDT CMFWS.
238+
239+
::
240+
241+
"decoders:root0":[
242+
{
243+
"decoder":"decoder0.0",
244+
"resource":825975898112,
245+
"size":274877906944,
246+
"interleave_ways":1,
247+
"max_available_extent":0,
248+
"volatile_capable":true,
249+
"nr_targets":2,
250+
"targets":[
251+
{
252+
"target":"pci0000:a8",
253+
"alias":"ACPI0016:02",
254+
"position":1,
255+
"id":4
256+
},
257+
],
258+
259+
Finally we have the `Memory Region` associated with the `Root Decoder`
260+
:code:`decoder0.0`. This region describes the overall interleave configuration
261+
of the interleave set.
262+
263+
::
264+
265+
"regions:decoder0.0":[
266+
{
267+
"region":"region0",
268+
"resource":825975898112,
269+
"size":274877906944,
270+
"type":"ram",
271+
"interleave_ways":2,
272+
"interleave_granularity":256,
273+
"decode_state":"commit",
274+
"mappings":[
275+
{
276+
"position":1,
277+
"memdev":"mem1",
278+
"decoder":"decoder6.0"
279+
},
280+
{
281+
"position":0,
282+
"memdev":"mem0",
283+
"decoder":"decoder5.0"
284+
}
285+
]
286+
}
287+
]
288+
}
289+
]
290+
}
291+
]

0 commit comments

Comments
 (0)