Skip to content

ubuntu server can see all GID list #164

New issue
New issue
Open
Open
ubuntu server can see all GID list#164
@SecurityNeo

Description

@SecurityNeo
SecurityNeo
opened on Mar 25, 2025

I run ubuntu pod in the cluster，in the container，i can see all gid list。

root@aigc-maas-instance-leader-0:/# ibv_devinfo -d mlx5_0 -v
hca_id: mlx5_0
transport: InfiniBand (0)
fw_ver: 14.29.1016
node_guid: c4c0:6300:00b7:5e76
sys_image_guid: c4c0:6300:00b7:5e76
vendor_id: 0x02c9
vendor_part_id: 4117
hw_ver: 0x0
board_id: H3C0010110034
phys_port_cnt: 1
max_mr_size: 0xffffffffffffffff
page_size_cap: 0xfffffffffffff000
max_qp: 262144
max_qp_wr: 32768
device_cap_flags: 0x25321c36
BAD_PKEY_CNTR
BAD_QKEY_CNTR
AUTO_PATH_MIG
CHANGE_PHY_PORT
PORT_ACTIVE_EVENT
SYS_IMAGE_GUID
RC_RNR_NAK_GEN
MEM_WINDOW
XRC
MEM_MGT_EXTENSIONS
MEM_WINDOW_TYPE_2B
RAW_IP_CSUM
MANAGED_FLOW_STEERING
max_sge: 30
max_sge_rd: 30
max_cq: 16777216
max_cqe: 4194303
max_mr: 16777216
max_pd: 16777216
max_qp_rd_atom: 16
max_ee_rd_atom: 0
max_res_rd_atom: 4194304
max_qp_init_rd_atom: 16
max_ee_init_rd_atom: 0
atomic_cap: ATOMIC_HCA (1)
max_ee: 0
max_rdd: 0
max_mw: 16777216
max_raw_ipv6_qp: 0
max_raw_ethy_qp: 0
max_mcast_grp: 2097152
max_mcast_qp_attach: 240
max_total_mcast_qp_attach: 503316480
max_ah: 2147483647
max_fmr: 0
max_srq: 8388608
max_srq_wr: 32767
max_srq_sge: 31
max_pkeys: 128
local_ca_ack_delay: 16
general_odp_caps:
ODP_SUPPORT
ODP_SUPPORT_IMPLICIT
rc_odp_caps:
SUPPORT_SEND
SUPPORT_RECV
SUPPORT_WRITE
SUPPORT_READ
SUPPORT_SRQ
uc_odp_caps:
NO SUPPORT
ud_odp_caps:
SUPPORT_SEND
xrc_odp_caps:
SUPPORT_SEND
SUPPORT_WRITE
SUPPORT_READ
SUPPORT_SRQ
completion timestamp_mask: 0x7fffffffffffffff
hca_core_clock: 156250kHZ
raw packet caps:
C-VLAN stripping offload
Scatter FCS offload
IP csum offload
Delay drop
device_cap_flags_ex: 0x2000001425321C36
RAW_SCATTER_FCS
PCI_WRITE_END_PADDING
Unknown flags: 0x2000000000000000
tso_caps:
max_tso: 262144
supported_qp:
SUPPORT_RAW_PACKET
rss_caps:
max_rwq_indirection_tables: 1048576
max_rwq_indirection_table_size: 2048
rx_hash_function: 0x1
rx_hash_fields_mask: 0x800000FF
supported_qp:
SUPPORT_RAW_PACKET
max_wq_type_rq: 8388608
packet_pacing_caps:
qp_rate_limit_min: 0kbps
qp_rate_limit_max: 0kbps
tag matching not supported

    cq moderation caps:
            max_cq_count:   65535
            max_cq_period:  4095 us

    num_comp_vectors:               63
            port:   1
                    state:                  PORT_ACTIVE (4)
                    max_mtu:                4096 (5)
                    active_mtu:             4096 (5)
                    sm_lid:                 0
                    port_lid:               0
                    port_lmc:               0x00
                    link_layer:             Ethernet
                    max_msg_sz:             0x40000000
                    port_cap_flags:         0x04010000
                    port_cap_flags2:        0x0000
                    max_vl_num:             invalid value (0)
                    bad_pkey_cntr:          0x0
                    qkey_viol_cntr:         0x0
                    sm_sl:                  0
                    pkey_tbl_len:           1
                    gid_tbl_len:            256
                    subnet_timeout:         0
                    init_type_reply:        0
                    active_width:           1X (1)
                    active_speed:           25.0 Gbps (32)
                    phys_state:             LINK_UP (5)
                    **GID[  0]:               fe80:0000:0000:0000:c6c0:63ff:feb7:5e76, RoCE v1
                    GID[  1]:               fe80::c6c0:63ff:feb7:5e76, RoCE v2
                    GID[  2]:               0000:0000:0000:0000:0000:ffff:c0a8:0603, RoCE v1
                    GID[  3]:               ::ffff:192.168.6.3, RoCE v2
                    GID[  4]:               fe80:0000:0000:0000:5c3a:85ff:fe96:3691, RoCE v1
                    GID[  5]:               fe80::5c3a:85ff:fe96:3691, RoCE v2
                    GID[  6]:               0000:0000:0000:0000:0000:ffff:c0a8:0604, RoCE v1
                    GID[  7]:               ::ffff:192.168.6.4, RoCE v2
                    GID[  8]:               fe80:0000:0000:0000:5c1f:b7ff:fe4a:6ea7, RoCE v1
                    GID[  9]:               fe80::5c1f:b7ff:fe4a:6ea7, RoCE v2
                    GID[ 10]:               0000:0000:0000:0000:0000:ffff:c0a8:0605, RoCE v1
                    GID[ 11]:               ::ffff:192.168.6.5, RoCE v2
                    GID[ 12]:               fe80:0000:0000:0000:b007:89ff:fe65:cd91, RoCE v1
                    GID[ 13]:               fe80::b007:89ff:fe65:cd91, RoCE v2
                    GID[ 14]:               0000:0000:0000:0000:0000:ffff:c0a8:0606, RoCE v1
                    GID[ 15]:               ::ffff:192.168.6.6, RoCE v2
                    GID[ 16]:               fe80:0000:0000:0000:4cbe:9aff:feeb:6b0f, RoCE v1
                    GID[ 17]:               fe80::4cbe:9aff:feeb:6b0f, RoCE v2
                    GID[ 18]:               0000:0000:0000:0000:0000:ffff:0303:0303, RoCE v1
                    GID[ 19]:               ::ffff:3.3.3.3, RoCE v2**

root@aigc-maas-instance-leader-0:/# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: net1@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default
link/ether b2:07:89:65:cd:91 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.6.5/24 brd 192.168.6.255 scope global net1
valid_lft forever preferred_lft forever
inet6 fe80::b007:89ff:fe65:cd91/64 scope link
valid_lft forever preferred_lft forever
26696: eth0@if26697: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1400 qdisc noqueue state UP group default
link/ether aa:9a:d6:60:7b:96 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 110.64.115.181/10 brd 110.127.255.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::a89a:d6ff:fe60:7b96/64 scope link
valid_lft forever preferred_lft forever

root@aigc-maas-instance-leader-0:/# cat /etc/os-release
PRETTY_NAME="Ubuntu 22.04.3 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.3 LTS (Jammy Jellyfish)"
VERSION_CODENAME=jammy
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=jammy

if i run the pod with image “mellanox/mofed:23.10-1.1.9.0-ubuntu22.04-amd64”，only what belongs to it will be displayed，of the two k8s yaml are the same。

[root@mellanox-my-test1-7db4744f55-vpvzn /]# ibv_devinfo -d mlx5_0 -v
hca_id: mlx5_0
transport: InfiniBand (0)
fw_ver: 14.29.1016
node_guid: c4c0:6300:00b7:5e76
sys_image_guid: c4c0:6300:00b7:5e76
vendor_id: 0x02c9
vendor_part_id: 4117
hw_ver: 0x0
board_id: H3C0010110034
phys_port_cnt: 1
max_mr_size: 0xffffffffffffffff
page_size_cap: 0xfffffffffffff000
max_qp: 262144
max_qp_wr: 32768
device_cap_flags: 0x25321c36
BAD_PKEY_CNTR
BAD_QKEY_CNTR
AUTO_PATH_MIG
CHANGE_PHY_PORT
PORT_ACTIVE_EVENT
SYS_IMAGE_GUID
RC_RNR_NAK_GEN
MEM_WINDOW
XRC
MEM_MGT_EXTENSIONS
MEM_WINDOW_TYPE_2B
RAW_IP_CSUM
MANAGED_FLOW_STEERING
max_sge: 30
max_sge_rd: 30
max_cq: 16777216
max_cqe: 4194303
max_mr: 16777216
max_pd: 16777216
max_qp_rd_atom: 16
max_ee_rd_atom: 0
max_res_rd_atom: 4194304
max_qp_init_rd_atom: 16
max_ee_init_rd_atom: 0
atomic_cap: ATOMIC_HCA (1)
max_ee: 0
max_rdd: 0
max_mw: 16777216
max_raw_ipv6_qp: 0
max_raw_ethy_qp: 0
max_mcast_grp: 2097152
max_mcast_qp_attach: 240
max_total_mcast_qp_attach: 503316480
max_ah: 2147483647
max_fmr: 0
max_srq: 8388608
max_srq_wr: 32767
max_srq_sge: 31
max_pkeys: 128
local_ca_ack_delay: 16
general_odp_caps:
ODP_SUPPORT
Unknown flags: 0x2
rc_odp_caps:
SUPPORT_SEND
SUPPORT_RECV
SUPPORT_WRITE
SUPPORT_READ
Unknown flags: 0x20
uc_odp_caps:
NO SUPPORT
ud_odp_caps:
SUPPORT_SEND
completion timestamp_mask: 0x7fffffffffffffff
hca_core_clock: 156250kHZ
raw packet caps:
C-VLAN stripping offload
Scatter FCS offload
IP csum offload
Delay drop
device_cap_flags_ex: 0x2000001425321C36
RAW_SCATTER_FCS
PCI_WRITE_END_PADDING
Unknown flags: 0x2000000000000000
tso_caps:
max_tso: 262144
supported_qp:
SUPPORT_RAW_PACKET
rss_caps:
max_rwq_indirection_tables: 1048576
max_rwq_indirection_table_size: 2048
rx_hash_function: 0x1
rx_hash_fields_mask: 0x800000FF
supported_qp:
SUPPORT_RAW_PACKET
max_wq_type_rq: 8388608
packet_pacing_caps:
qp_rate_limit_min: 0kbps
qp_rate_limit_max: 0kbps
tag matching not supported

    cq moderation caps:
            max_cq_count:   65535
            max_cq_period:  4095 us

            port:   1
                    state:                  PORT_ACTIVE (4)
                    max_mtu:                4096 (5)
                    active_mtu:             4096 (5)
                    sm_lid:                 0
                    port_lid:               0
                    port_lmc:               0x00
                    link_layer:             Ethernet
                    max_msg_sz:             0x40000000
                    port_cap_flags:         0x04010000
                    port_cap_flags2:        0x0000
                    max_vl_num:             invalid value (0)
                    bad_pkey_cntr:          0x0
                    qkey_viol_cntr:         0x0
                    sm_sl:                  0
                    pkey_tbl_len:           1
                    gid_tbl_len:            256
                    subnet_timeout:         0
                    init_type_reply:        0
                    active_width:           1X (1)
                    active_speed:           25.0 Gbps (32)
                    phys_state:             LINK_UP (5)
                    **GID[ 14]:               0000:0000:0000:0000:0000:ffff:c0a8:0606
                    GID[ 15]:               0000:0000:0000:0000:0000:ffff:c0a8:0606
                    GID[ 16]:               fe80:0000:0000:0000:4cbe:9aff:feeb:6b0f
                    GID[ 17]:               fe80:0000:0000:0000:4cbe:9aff:feeb:6b0f**

[root@mellanox-my-test1-7db4744f55-vpvzn /]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: net1@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default
link/ether 4e:be:9a:eb:6b:0f brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.6.6/24 brd 192.168.6.255 scope global net1
valid_lft forever preferred_lft forever
inet6 fe80::4cbe:9aff:feeb:6b0f/64 scope link
valid_lft forever preferred_lft forever
3: net2@if5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default
link/ether 72:7a:e9:65:3a:e5 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.7.4/24 brd 192.168.7.255 scope global net2
valid_lft forever preferred_lft forever
inet6 fe80::707a:e9ff:fe65:3ae5/64 scope link
valid_lft forever preferred_lft forever
4: net3@if7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default
link/ether 26:bf:21:2f:74:d6 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.8.4/24 brd 192.168.8.255 scope global net3
valid_lft forever preferred_lft forever
inet6 fe80::24bf:21ff:fe2f:74d6/64 scope link
valid_lft forever preferred_lft forever
5: net4@if6: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default
link/ether 3e:16:f0:d5:59:6d brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.9.4/24 brd 192.168.9.255 scope global net4
valid_lft forever preferred_lft forever
inet6 fe80::3c16:f0ff:fed5:596d/64 scope link
valid_lft forever preferred_lft forever
2915: eth0@if2916: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1400 qdisc noqueue state UP group default
link/ether 5a:1f:3d:f6:4b:a1 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 110.64.64.191/10 brd 110.127.255.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::581f:3dff:fef6:4ba1/64 scope link
valid_lft forever preferred_lft forever
[root@mellanox-my-test1-7db4744f55-vpvzn /]# cat /etc/os-release
NAME="CentOS Linux"
VERSION="7 (Core)"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="7"
PRETTY_NAME="CentOS Linux 7 (Core)"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:7"
HOME_URL="https://www.centos.org/"
BUG_REPORT_URL="https://bugs.centos.org/"

CENTOS_MANTISBT_PROJECT="CentOS-7"
CENTOS_MANTISBT_PROJECT_VERSION="7"
REDHAT_SUPPORT_PRODUCT="centos"
REDHAT_SUPPORT_PRODUCT_VERSION="7"

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions