ib_mi.c File Reference

Infiniband management interfaces. More...

#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <byteswap.h>
#include <gpxe/infiniband.h>
#include <gpxe/iobuf.h>
#include <gpxe/ib_mi.h>

Go to the source code of this file.

Defines

#define IB_MI_NUM_SEND_WQES   4
 Management interface number of send WQEs.
#define IB_MI_NUM_RECV_WQES   2
 Management interface number of receive WQEs.
#define IB_MI_NUM_CQES   8
 Management interface number of completion queue entries.
#define IB_MI_TID_MAGIC   ( ( 'g' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )
 TID magic signature.

Functions

 FILE_LICENCE (GPL2_OR_LATER)
static int ib_mi_handle (struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av)
 Handle received MAD.
static void ib_mi_complete_recv (struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *av, struct io_buffer *iobuf, int rc)
 Complete receive via management interface.
int ib_mi_send (struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av)
 Transmit MAD.
static void ib_mi_timer_expired (struct retry_timer *timer, int expired)
 Handle management transaction timer expiry.
struct ib_mad_transaction * ib_create_madx (struct ib_device *ibdev, struct ib_mad_interface *mi, union ib_mad *mad, struct ib_address_vector *av, struct ib_mad_transaction_operations *op)
 Create management transaction.
void ib_destroy_madx (struct ib_device *ibdev __unused, struct ib_mad_interface *mi __unused, struct ib_mad_transaction *madx)
 Destroy management transaction.
struct ib_mad_interface * ib_create_mi (struct ib_device *ibdev, enum ib_queue_pair_type type)
 Create management interface.
void ib_destroy_mi (struct ib_device *ibdev, struct ib_mad_interface *mi)
 Destroy management interface.

Variables

static unsigned int next_tid
 TID to use for next MAD.
static struct
ib_completion_queue_operations 		ib_mi_completion_ops
 Management interface completion operations.

Detailed Description

Infiniband management interfaces.

Definition in file ib_mi.c.

Define Documentation

#define IB_MI_NUM_SEND_WQES   4

Management interface number of send WQEs.

This is a policy decision.

Definition at line 43 of file ib_mi.c.

Referenced by ib_create_mi().

#define IB_MI_NUM_RECV_WQES   2

Management interface number of receive WQEs.

This is a policy decision.

Definition at line 49 of file ib_mi.c.

Referenced by ib_create_mi().

#define IB_MI_NUM_CQES   8

Management interface number of completion queue entries.

This is a policy decision

Definition at line 55 of file ib_mi.c.

Referenced by ib_create_mi().

#define IB_MI_TID_MAGIC   ( ( 'g' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )

TID magic signature.

Definition at line 58 of file ib_mi.c.

Referenced by ib_mi_send().

Function Documentation

FILE_LICENCE ( GPL2_OR_LATER   ) 

static int ib_mi_handle ( struct ib_device *  ibdev,
struct ib_mad_interface *  mi,
union ib_mad *  mad,
struct ib_address_vector *  av 
) [static]

Handle received MAD.

Parameters:
ibdev Infiniband device
mi Management interface
mad Received MAD
av Source address vector
Return values:
rc Return status code

Definition at line 72 of file ib_mi.c.

References ib_mad_hdr::attr_id, ib_mad_agent::attr_id, ib_mad_hdr::class_version, ib_mad_agent::class_version, ib_mad_transaction_operations::complete, DBGC, ENOTSUP, for_each_table_entry, ib_mad_agent::handle, ib_mad::hdr, IB_MAD_AGENTS, IB_MGMT_CLASS_MASK, list_for_each_entry, ib_mad_transaction::mad, ib_mad_interface::madx, memcmp(), ib_mad_hdr::mgmt_class, ib_mad_agent::mgmt_class, ntohl, ib_mad_transaction::op, and ib_mad_hdr::tid.

Referenced by ib_mi_complete_recv().

00075                                                          {
00076         struct ib_mad_hdr *hdr = &mad->hdr;
00077         struct ib_mad_transaction *madx;
00078         struct ib_mad_agent *agent;
00079 
00080         /* Look for a matching transaction by TID */
00081         list_for_each_entry ( madx, &mi->madx, list ) {
00082                 if ( memcmp ( &hdr->tid, &madx->mad.hdr.tid,
00083                               sizeof ( hdr->tid ) ) != 0 )
00084                         continue;
00085                 /* Found a matching transaction */
00086                 madx->op->complete ( ibdev, mi, madx, 0, mad, av );
00087                 return 0;
00088         }
00089 
00090         /* If there is no matching transaction, look for a listening agent */
00091         for_each_table_entry ( agent, IB_MAD_AGENTS ) {
00092                 if ( ( ( agent->mgmt_class & IB_MGMT_CLASS_MASK ) !=
00093                        ( hdr->mgmt_class & IB_MGMT_CLASS_MASK ) ) ||
00094                      ( agent->class_version != hdr->class_version ) ||
00095                      ( agent->attr_id != hdr->attr_id ) )
00096                         continue;
00097                 /* Found a matching agent */
00098                 agent->handle ( ibdev, mi, mad, av );
00099                 return 0;
00100         }
00101 
00102         /* Otherwise, ignore it */
00103         DBGC ( mi, "MI %p RX TID %08x%08x ignored\n",
00104                mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) );
00105         return -ENOTSUP;
00106 }

static void ib_mi_complete_recv ( struct ib_device *  ibdev,
struct ib_queue_pair *  qp,
struct ib_address_vector *  av,
struct io_buffer *  iobuf,
int  rc 
) [static]

Complete receive via management interface.

Parameters:
ibdev Infiniband device
qp Queue pair
av Address vector
iobuf I/O buffer
rc Completion status code

Definition at line 118 of file ib_mi.c.

References ib_mad_hdr::attr_id, ib_mad_hdr::base_version, ib_mad_hdr::class_version, io_buffer::data, DBGC, DBGC2_HDA, DBGC_HDA, free_iob(), ib_mad::hdr, IB_MGMT_BASE_VERSION, ib_mi_handle(), ib_qp_get_ownerdata(), iob_len(), ib_mad_hdr::method, ib_mad_hdr::mgmt_class, ntohl, ntohs, ib_mad_hdr::status, strerror(), and ib_mad_hdr::tid.

00121                                                                     {
00122         struct ib_mad_interface *mi = ib_qp_get_ownerdata ( qp );
00123         union ib_mad *mad;
00124         struct ib_mad_hdr *hdr;
00125 
00126         /* Ignore errors */
00127         if ( rc != 0 ) {
00128                 DBGC ( mi, "MI %p RX error: %s\n", mi, strerror ( rc ) );
00129                 goto out;
00130         }
00131 
00132         /* Sanity checks */
00133         if ( iob_len ( iobuf ) != sizeof ( *mad ) ) {
00134                 DBGC ( mi, "MI %p RX bad size (%zd bytes)\n",
00135                        mi, iob_len ( iobuf ) );
00136                 DBGC_HDA ( mi, 0, iobuf->data, iob_len ( iobuf ) );
00137                 goto out;
00138         }
00139         mad = iobuf->data;
00140         hdr = &mad->hdr;
00141         if ( hdr->base_version != IB_MGMT_BASE_VERSION ) {
00142                 DBGC ( mi, "MI %p RX unsupported base version %x\n",
00143                        mi, hdr->base_version );
00144                 DBGC_HDA ( mi, 0, mad, sizeof ( *mad ) );
00145                 goto out;
00146         }
00147         DBGC ( mi, "MI %p RX TID %08x%08x (%02x,%02x,%02x,%04x) status "
00148                "%04x\n", mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
00149                hdr->mgmt_class, hdr->class_version, hdr->method,
00150                ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
00151         DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );
00152 
00153         /* Handle MAD */
00154         if ( ( rc = ib_mi_handle ( ibdev, mi, mad, av ) ) != 0 )
00155                 goto out;
00156 
00157  out:
00158         free_iob ( iobuf );
00159 }

int ib_mi_send ( struct ib_device *  ibdev,
struct ib_mad_interface *  mi,
union ib_mad *  mad,
struct ib_address_vector *  av 
)

Transmit MAD.

Parameters:
ibdev Infiniband device
mi Management interface
mad MAD
av Destination address vector
Return values:
rc Return status code

Definition at line 175 of file ib_mi.c.

References alloc_iob(), assert, ib_mad_hdr::attr_id, ib_mad_hdr::base_version, ib_mad_hdr::class_specific, ib_mad_hdr::class_version, DBGC, DBGC2_HDA, EINVAL, ENOMEM, free_iob(), ib_mad::hdr, ib_smp_class_specific::hop_count, ib_smp_class_specific::hop_pointer, ib_smp_dr_path::hops, htonl, htons, IB_MGMT_BASE_VERSION, IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE, IB_MI_TID_MAGIC, ib_post_send(), IB_SMP_STATUS_D_INBOUND, iob_put, ib_mad_smp::mad_hdr, memcpy, ib_mad_hdr::method, ib_mad_hdr::mgmt_class, next_tid, ntohl, ntohs, ib_device::port, ib_mad_interface::qp, ib_mad_smp::return_path, ib_mad_class_specific::smp, ib_mad::smp, ib_mad_hdr::status, strerror(), and ib_mad_hdr::tid.

Referenced by ib_cm_send_rtu(), ib_mcast_leave(), ib_mi_timer_expired(), ib_sma_guid_info(), ib_sma_node_desc(), ib_sma_node_info(), ib_sma_pkey_table(), and ib_sma_port_info().

00176                                                                    {
00177         struct ib_mad_hdr *hdr = &mad->hdr;
00178         struct io_buffer *iobuf;
00179         int rc;
00180 
00181         /* Set common fields */
00182         hdr->base_version = IB_MGMT_BASE_VERSION;
00183         if ( ( hdr->tid[0] == 0 ) && ( hdr->tid[1] == 0 ) ) {
00184                 hdr->tid[0] = htonl ( IB_MI_TID_MAGIC );
00185                 hdr->tid[1] = htonl ( ++next_tid );
00186         }
00187         DBGC ( mi, "MI %p TX TID %08x%08x (%02x,%02x,%02x,%04x) status "
00188                "%04x\n", mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
00189                hdr->mgmt_class, hdr->class_version, hdr->method,
00190                ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
00191         DBGC2_HDA ( mi, 0, mad, sizeof ( *mad ) );
00192 
00193         /* Construct directed route portion of response, if necessary */
00194         if ( hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ) {
00195                 struct ib_mad_smp *smp = &mad->smp;
00196                 unsigned int hop_pointer;
00197                 unsigned int hop_count;
00198 
00199                 smp->mad_hdr.status |= htons ( IB_SMP_STATUS_D_INBOUND );
00200                 hop_pointer = smp->mad_hdr.class_specific.smp.hop_pointer;
00201                 hop_count = smp->mad_hdr.class_specific.smp.hop_count;
00202                 assert ( hop_count == hop_pointer );
00203                 if ( hop_pointer < ( sizeof ( smp->return_path.hops ) /
00204                                      sizeof ( smp->return_path.hops[0] ) ) ) {
00205                         smp->return_path.hops[hop_pointer] = ibdev->port;
00206                 } else {
00207                         DBGC ( mi, "MI %p TX TID %08x%08x invalid hop pointer "
00208                                "%d\n", mi, ntohl ( hdr->tid[0] ),
00209                                ntohl ( hdr->tid[1] ), hop_pointer );
00210                         return -EINVAL;
00211                 }
00212         }
00213 
00214         /* Construct I/O buffer */
00215         iobuf = alloc_iob ( sizeof ( *mad ) );
00216         if ( ! iobuf ) {
00217                 DBGC ( mi, "MI %p could not allocate buffer for TID "
00218                        "%08x%08x\n",
00219                        mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) );
00220                 return -ENOMEM;
00221         }
00222         memcpy ( iob_put ( iobuf, sizeof ( *mad ) ), mad, sizeof ( *mad ) );
00223 
00224         /* Send I/O buffer */
00225         if ( ( rc = ib_post_send ( ibdev, mi->qp, av, iobuf ) ) != 0 ) {
00226                 DBGC ( mi, "MI %p TX TID %08x%08x failed: %s\n",
00227                        mi,  ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
00228                        strerror ( rc ) );
00229                 free_iob ( iobuf );
00230                 return rc;
00231         }
00232 
00233         return 0;
00234 }

static void ib_mi_timer_expired ( struct retry_timer *  timer,
int  expired 
) [static]

Handle management transaction timer expiry.

Parameters:
timer Retry timer
expired Failure indicator

Definition at line 242 of file ib_mi.c.

References ib_mad_transaction::av, ib_mad_transaction_operations::complete, container_of, DBGC, ETIMEDOUT, ib_mad::hdr, ib_mi_send(), ib_mad_interface::ibdev, ib_mad_transaction::mad, ib_mad_transaction::mi, ntohl, NULL, ib_mad_transaction::op, start_timer(), and ib_mad_hdr::tid.

Referenced by ib_create_madx().

00242                                                                            {
00243         struct ib_mad_transaction *madx =
00244                 container_of ( timer, struct ib_mad_transaction, timer );
00245         struct ib_mad_interface *mi = madx->mi;
00246         struct ib_device *ibdev = mi->ibdev;
00247         struct ib_mad_hdr *hdr = &madx->mad.hdr;
00248 
00249         /* Abandon transaction if we have tried too many times */
00250         if ( expired ) {
00251                 DBGC ( mi, "MI %p abandoning TID %08x%08x\n",
00252                        mi, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ) );
00253                 madx->op->complete ( ibdev, mi, madx, -ETIMEDOUT, NULL, NULL );
00254                 return;
00255         }
00256 
00257         /* Restart retransmission timer */
00258         start_timer ( timer );
00259 
00260         /* Resend MAD */
00261         ib_mi_send ( ibdev, mi, &madx->mad, &madx->av );
00262 }

struct ib_mad_transaction* ib_create_madx ( struct ib_device *  ibdev,
struct ib_mad_interface *  mi,
union ib_mad *  mad,
struct ib_address_vector *  av,
struct ib_mad_transaction_operations *  op 
) [read]

Create management transaction.

Parameters:
ibdev Infiniband device
mi Management interface
mad MAD to send
av Destination address, or NULL to use SM's GSI
op Management transaction operations
Return values:
madx Management transaction, or NULL

Definition at line 275 of file ib_mi.c.

References ib_mad_transaction::av, retry_timer::expired, ib_mi_timer_expired(), IB_QKEY_GSI, IB_QPN_GSI, ib_address_vector::lid, ib_mad_transaction::list, list_add, ib_mad_transaction::mad, ib_mad_interface::madx, memcpy, ib_mad_transaction::mi, NULL, ib_mad_transaction::op, ib_address_vector::qkey, ib_address_vector::qpn, ib_address_vector::sl, ib_device::sm_lid, ib_device::sm_sl, start_timer_nodelay(), ib_mad_transaction::timer, and zalloc().

Referenced by ib_cm_path_complete(), ib_create_path(), and ib_mcast_join().

00277                                                             {
00278         struct ib_mad_transaction *madx;
00279 
00280         /* Allocate and initialise structure */
00281         madx = zalloc ( sizeof ( *madx ) );
00282         if ( ! madx )
00283                 return NULL;
00284         madx->mi = mi;
00285         madx->timer.expired = ib_mi_timer_expired;
00286         madx->op = op;
00287 
00288         /* Determine address vector */
00289         if ( av ) {
00290                 memcpy ( &madx->av, av, sizeof ( madx->av ) );
00291         } else {
00292                 madx->av.lid = ibdev->sm_lid;
00293                 madx->av.sl = ibdev->sm_sl;
00294                 madx->av.qpn = IB_QPN_GSI;
00295                 madx->av.qkey = IB_QKEY_GSI;
00296         }
00297 
00298         /* Copy MAD */
00299         memcpy ( &madx->mad, mad, sizeof ( madx->mad ) );
00300 
00301         /* Add to list and start timer to send initial MAD */
00302         list_add ( &madx->list, &mi->madx );
00303         start_timer_nodelay ( &madx->timer );
00304 
00305         return madx;
00306 }

void ib_destroy_madx ( struct ib_device *ibdev  __unused,
struct ib_mad_interface *mi  __unused,
struct ib_mad_transaction *  madx 
)

Destroy management transaction.

Parameters:
ibdev Infiniband device
mi Management interface
madx Management transaction

Definition at line 315 of file ib_mi.c.

References free(), ib_mad_transaction::list, list_del, stop_timer(), and ib_mad_transaction::timer.

Referenced by ib_cm_req_complete(), ib_create_path(), ib_destroy_conn(), ib_destroy_path(), ib_mcast_complete(), ib_mcast_join(), ib_mcast_leave(), and ib_path_complete().

00317                                                          {
00318 
00319         /* Stop timer and remove from list */
00320         stop_timer ( &madx->timer );
00321         list_del ( &madx->list );
00322 
00323         /* Free transaction */
00324         free ( madx );
00325 }

struct ib_mad_interface* ib_create_mi ( struct ib_device *  ibdev,
enum ib_queue_pair_type  type 
) [read]

Create management interface.

Parameters:
ibdev Infiniband device
type Queue pair type
Return values:
mi Management agent, or NULL

Definition at line 334 of file ib_mi.c.

References ib_mad_interface::cq, DBGC, free(), ib_create_cq(), ib_create_qp(), ib_destroy_cq(), ib_destroy_qp(), IB_MI_NUM_CQES, IB_MI_NUM_RECV_WQES, IB_MI_NUM_SEND_WQES, ib_modify_qp(), IB_QKEY_GSI, IB_QKEY_SMI, ib_qp_set_ownerdata(), IB_QPT_SMI, ib_refill_recv(), ib_mad_interface::ibdev, INIT_LIST_HEAD, ib_mad_interface::madx, NULL, ib_queue_pair::qkey, ib_mad_interface::qp, ib_queue_pair::qpn, strerror(), and zalloc().

Referenced by ib_open().

00335                                                                         {
00336         struct ib_mad_interface *mi;
00337         int rc;
00338 
00339         /* Allocate and initialise fields */
00340         mi = zalloc ( sizeof ( *mi ) );
00341         if ( ! mi )
00342                 goto err_alloc;
00343         mi->ibdev = ibdev;
00344         INIT_LIST_HEAD ( &mi->madx );
00345 
00346         /* Create completion queue */
00347         mi->cq = ib_create_cq ( ibdev, IB_MI_NUM_CQES, &ib_mi_completion_ops );
00348         if ( ! mi->cq ) {
00349                 DBGC ( mi, "MI %p could not allocate completion queue\n", mi );
00350                 goto err_create_cq;
00351         }
00352 
00353         /* Create queue pair */
00354         mi->qp = ib_create_qp ( ibdev, type, IB_MI_NUM_SEND_WQES, mi->cq,
00355                                 IB_MI_NUM_RECV_WQES, mi->cq );
00356         if ( ! mi->qp ) {
00357                 DBGC ( mi, "MI %p could not allocate queue pair\n", mi );
00358                 goto err_create_qp;
00359         }
00360         ib_qp_set_ownerdata ( mi->qp, mi );
00361         DBGC ( mi, "MI %p (%s) running on QPN %#lx\n",
00362                mi, ( ( type == IB_QPT_SMI ) ? "SMI" : "GSI" ), mi->qp->qpn );
00363 
00364         /* Set queue key */
00365         mi->qp->qkey = ( ( type == IB_QPT_SMI ) ? IB_QKEY_SMI : IB_QKEY_GSI );
00366         if ( ( rc = ib_modify_qp ( ibdev, mi->qp ) ) != 0 ) {
00367                 DBGC ( mi, "MI %p could not set queue key: %s\n",
00368                        mi, strerror ( rc ) );
00369                 goto err_modify_qp;
00370         }
00371 
00372         /* Fill receive ring */
00373         ib_refill_recv ( ibdev, mi->qp );
00374         return mi;
00375 
00376  err_modify_qp:
00377         ib_destroy_qp ( ibdev, mi->qp );
00378  err_create_qp:
00379         ib_destroy_cq ( ibdev, mi->cq );
00380  err_create_cq:
00381         free ( mi );
00382  err_alloc:
00383         return NULL;
00384 }

void ib_destroy_mi ( struct ib_device *  ibdev,
struct ib_mad_interface *  mi 
)

Destroy management interface.

Parameters:
mi Management interface

Definition at line 391 of file ib_mi.c.

References ib_mad_transaction_operations::complete, ib_mad_interface::cq, DBGC, ECANCELED, free(), ib_mad::hdr, ib_destroy_cq(), ib_destroy_qp(), ib_mad_transaction::list, list_for_each_entry_safe, ib_mad_transaction::mad, ib_mad_interface::madx, ntohl, NULL, ib_mad_transaction::op, ib_mad_interface::qp, and ib_mad_hdr::tid.

Referenced by ib_close(), and ib_open().

00391                                                                             {
00392         struct ib_mad_transaction *madx;
00393         struct ib_mad_transaction *tmp;
00394 
00395         /* Flush any outstanding requests */
00396         list_for_each_entry_safe ( madx, tmp, &mi->madx, list ) {
00397                 DBGC ( mi, "MI %p destroyed while TID %08x%08x in progress\n",
00398                        mi, ntohl ( madx->mad.hdr.tid[0] ),
00399                        ntohl ( madx->mad.hdr.tid[1] ) );
00400                 madx->op->complete ( ibdev, mi, madx, -ECANCELED, NULL, NULL );
00401         }
00402 
00403         ib_destroy_qp ( ibdev, mi->qp );
00404         ib_destroy_cq ( ibdev, mi->cq );
00405         free ( mi );
00406 }

Variable Documentation

unsigned int next_tid [static]

TID to use for next MAD.

Definition at line 61 of file ib_mi.c.

Referenced by ib_mi_send().

struct ib_completion_queue_operations ib_mi_completion_ops [static]

Initial value:

 {
        .complete_recv = ib_mi_complete_recv,
}
Management interface completion operations.

Definition at line 162 of file ib_mi.c.

Generated on Tue Apr 6 20:01:55 2010 for gPXE by  doxygen 1.5.7.1