ib_cmrc.c File Reference

Infiniband Communication-managed Reliable Connections. More...

#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <gpxe/iobuf.h>
#include <gpxe/xfer.h>
#include <gpxe/process.h>
#include <gpxe/infiniband.h>
#include <gpxe/ib_cm.h>
#include <gpxe/ib_cmrc.h>

Go to the source code of this file.

Data Structures

struct  ib_cmrc_connection
 An Infiniband Communication-Managed Reliable Connection. More...

Defines

#define IB_CMRC_NUM_SEND_WQES   4
 CMRC number of send WQEs.
#define IB_CMRC_NUM_RECV_WQES   2
 CMRC number of receive WQEs.
#define IB_CMRC_NUM_CQES   8
 CMRC number of completion queue entries.

Functions

 FILE_LICENCE (BSD2)
static void ib_cmrc_shutdown (struct process *process)
 Shut down CMRC connection gracefully.
static void ib_cmrc_close (struct ib_cmrc_connection *cmrc, int rc)
 Close CMRC connection.
static void ib_cmrc_changed (struct ib_device *ibdev __unused, struct ib_queue_pair *qp, struct ib_connection *conn __unused, int rc_cm, void *private_data, size_t private_data_len)
 Handle change of CMRC connection status.
static void ib_cmrc_complete_send (struct ib_device *ibdev __unused, struct ib_queue_pair *qp, struct io_buffer *iobuf, int rc)
 Handle CMRC send completion.
static void ib_cmrc_complete_recv (struct ib_device *ibdev __unused, struct ib_queue_pair *qp, struct ib_address_vector *av __unused, struct io_buffer *iobuf, int rc)
 Handle CMRC receive completion.
static int ib_cmrc_xfer_deliver_iob (struct xfer_interface *xfer, struct io_buffer *iobuf, struct xfer_metadata *meta __unused)
 Send data via CMRC.
static size_t ib_cmrc_xfer_window (struct xfer_interface *xfer)
 Check CMRC flow control window.
static void ib_cmrc_xfer_close (struct xfer_interface *xfer, int rc)
 Close CMRC data-transfer interface.
int ib_cmrc_open (struct xfer_interface *xfer, struct ib_device *ibdev, struct ib_gid *dgid, struct ib_gid_half *service_id)
 Open CMRC connection.

Variables

static struct
ib_connection_operations 		ib_cmrc_conn_op
 CMRC connection operations.
static struct
ib_completion_queue_operations 		ib_cmrc_completion_ops
 Infiniband CMRC completion operations.
static struct
xfer_interface_operations 		ib_cmrc_xfer_operations
 CMRC data transfer interface operations.

Detailed Description

Infiniband Communication-managed Reliable Connections.

Definition in file ib_cmrc.c.

Define Documentation

#define IB_CMRC_NUM_SEND_WQES   4

CMRC number of send WQEs.

This is a policy decision.

Definition at line 54 of file ib_cmrc.c.

Referenced by ib_cmrc_open().

#define IB_CMRC_NUM_RECV_WQES   2

CMRC number of receive WQEs.

This is a policy decision.

Definition at line 60 of file ib_cmrc.c.

Referenced by ib_cmrc_open().

#define IB_CMRC_NUM_CQES   8

CMRC number of completion queue entries.

This is a policy decision

Definition at line 66 of file ib_cmrc.c.

Referenced by ib_cmrc_open().

Function Documentation

FILE_LICENCE ( BSD2   ) 

static void ib_cmrc_shutdown ( struct process *  process  )  [static]

Shut down CMRC connection gracefully.

Parameters:
process Process
The Infiniband data structures are not reference-counted or guarded. It is therefore unsafe to shut them down while we may be in the middle of a callback from the Infiniband stack (e.g. in a receive completion handler).

This shutdown process will run some time after the call to ib_cmrc_close(), after control has returned out of the Infiniband core, and will shut down the Infiniband interfaces cleanly.

The shutdown process holds an implicit reference on the CMRC connection, ensuring that the structure is not freed before the shutdown process has run.

Definition at line 110 of file ib_cmrc.c.

References ib_cmrc_connection::conn, container_of, ib_cmrc_connection::cq, DBGC, ib_close(), ib_destroy_conn(), ib_destroy_cq(), ib_destroy_qp(), ib_cmrc_connection::ibdev, process_del(), ib_cmrc_connection::qp, ref_put(), ib_cmrc_connection::refcnt, ib_cmrc_connection::shutdown, and shutdown().

Referenced by ib_cmrc_open().

00110                                                          {
00111         struct ib_cmrc_connection *cmrc =
00112                 container_of ( process, struct ib_cmrc_connection, shutdown );
00113 
00114         DBGC ( cmrc, "CMRC %p shutting down\n", cmrc );
00115 
00116         /* Shut down Infiniband interface */
00117         ib_destroy_conn ( cmrc->ibdev, cmrc->qp, cmrc->conn );
00118         ib_destroy_qp ( cmrc->ibdev, cmrc->qp );
00119         ib_destroy_cq ( cmrc->ibdev, cmrc->cq );
00120         ib_close ( cmrc->ibdev );
00121 
00122         /* Remove process from run queue */
00123         process_del ( &cmrc->shutdown );
00124 
00125         /* Drop the remaining reference */
00126         ref_put ( &cmrc->refcnt );
00127 }

static void ib_cmrc_close ( struct ib_cmrc_connection *  cmrc,
int  rc 
) [static]

Close CMRC connection.

Parameters:
cmrc Communication-Managed Reliable Connection
rc Reason for close

Definition at line 135 of file ib_cmrc.c.

References process_add(), ib_cmrc_connection::shutdown, ib_cmrc_connection::xfer, and xfer_close().

Referenced by ib_cmrc_changed(), ib_cmrc_complete_recv(), ib_cmrc_complete_send(), ib_cmrc_xfer_close(), and ib_cmrc_xfer_deliver_iob().

00135                                                                       {
00136 
00137         /* Close data transfer interface */
00138         xfer_nullify ( &cmrc->xfer );
00139         xfer_close ( &cmrc->xfer, rc );
00140 
00141         /* Schedule shutdown process */
00142         process_add ( &cmrc->shutdown );
00143 }

static void ib_cmrc_changed ( struct ib_device *ibdev  __unused,
struct ib_queue_pair *  qp,
struct ib_connection *conn  __unused,
int  rc_cm,
void *  private_data,
size_t  private_data_len 
) [static]

Handle change of CMRC connection status.

Parameters:
ibdev Infiniband device
qp Queue pair
conn Connection
rc_cm Connection status code
private_data Private data, if available
private_data_len Length of private data

Definition at line 155 of file ib_cmrc.c.

References ib_cmrc_connection::connected, DBGC, DBGC2, DBGC2_HDA, ib_cmrc_close(), ib_qp_get_ownerdata(), strerror(), ib_cmrc_connection::xfer, and xfer_deliver_raw().

00158                                                                             {
00159         struct ib_cmrc_connection *cmrc = ib_qp_get_ownerdata ( qp );
00160         int rc_xfer;
00161 
00162         /* Record connection status */
00163         if ( rc_cm == 0 ) {
00164                 DBGC ( cmrc, "CMRC %p connected\n", cmrc );
00165                 cmrc->connected = 1;
00166         } else {
00167                 DBGC ( cmrc, "CMRC %p disconnected: %s\n",
00168                        cmrc, strerror ( rc_cm ) );
00169                 cmrc->connected = 0;
00170         }
00171 
00172         /* Pass up any private data */
00173         DBGC2 ( cmrc, "CMRC %p received private data:\n", cmrc );
00174         DBGC2_HDA ( cmrc, 0, private_data, private_data_len );
00175         if ( private_data &&
00176              ( rc_xfer = xfer_deliver_raw ( &cmrc->xfer, private_data,
00177                                             private_data_len ) ) != 0 ) {
00178                 DBGC ( cmrc, "CMRC %p could not deliver private data: %s\n",
00179                        cmrc, strerror ( rc_xfer ) );
00180                 ib_cmrc_close ( cmrc, rc_xfer );
00181                 return;
00182         }
00183 
00184         /* If we are disconnected, close the upper connection */
00185         if ( rc_cm != 0 ) {
00186                 ib_cmrc_close ( cmrc, rc_cm );
00187                 return;
00188         }
00189 }

static void ib_cmrc_complete_send ( struct ib_device *ibdev  __unused,
struct ib_queue_pair *  qp,
struct io_buffer *  iobuf,
int  rc 
) [static]

Handle CMRC send completion.

Parameters:
ibdev Infiniband device
qp Queue pair
iobuf I/O buffer
rc Completion status code

Definition at line 204 of file ib_cmrc.c.

References DBGC, free_iob(), ib_cmrc_close(), ib_qp_get_ownerdata(), and strerror().

00206                                                                       {
00207         struct ib_cmrc_connection *cmrc = ib_qp_get_ownerdata ( qp );
00208 
00209         /* Free the completed I/O buffer */
00210         free_iob ( iobuf );
00211 
00212         /* Close the connection on any send errors */
00213         if ( rc != 0 ) {
00214                 DBGC ( cmrc, "CMRC %p send error: %s\n",
00215                        cmrc, strerror ( rc ) );
00216                 ib_cmrc_close ( cmrc, rc );
00217                 return;
00218         }
00219 }

static void ib_cmrc_complete_recv ( struct ib_device *ibdev  __unused,
struct ib_queue_pair *  qp,
struct ib_address_vector *av  __unused,
struct io_buffer *  iobuf,
int  rc 
) [static]

Handle CMRC receive completion.

Parameters:
ibdev Infiniband device
qp Queue pair
av Address vector, or NULL
iobuf I/O buffer
rc Completion status code

Definition at line 230 of file ib_cmrc.c.

References io_buffer::data, DBGC, DBGC2, DBGC2_HDA, free_iob(), ib_cmrc_close(), ib_qp_get_ownerdata(), iob_len(), strerror(), ib_cmrc_connection::xfer, and xfer_deliver_iob().

00233                                                                       {
00234         struct ib_cmrc_connection *cmrc = ib_qp_get_ownerdata ( qp );
00235 
00236         /* Close the connection on any receive errors */
00237         if ( rc != 0 ) {
00238                 DBGC ( cmrc, "CMRC %p receive error: %s\n",
00239                        cmrc, strerror ( rc ) );
00240                 free_iob ( iobuf );
00241                 ib_cmrc_close ( cmrc, rc );
00242                 return;
00243         }
00244 
00245         DBGC2 ( cmrc, "CMRC %p received:\n", cmrc );
00246         DBGC2_HDA ( cmrc, 0, iobuf->data, iob_len ( iobuf ) );
00247 
00248         /* Pass up data */
00249         if ( ( rc = xfer_deliver_iob ( &cmrc->xfer, iobuf ) ) != 0 ) {
00250                 DBGC ( cmrc, "CMRC %p could not deliver data: %s\n",
00251                        cmrc, strerror ( rc ) );
00252                 ib_cmrc_close ( cmrc, rc );
00253                 return;
00254         }
00255 }

static int ib_cmrc_xfer_deliver_iob ( struct xfer_interface *  xfer,
struct io_buffer *  iobuf,
struct xfer_metadata *meta  __unused 
) [static]

Send data via CMRC.

Parameters:
xfer Data transfer interface
iobuf Datagram I/O buffer
meta Data transfer metadata
Return values:
rc Return status code

Definition at line 271 of file ib_cmrc.c.

References ib_cmrc_connection::conn, ib_cmrc_connection::connected, container_of, io_buffer::data, DBGC, ib_cmrc_connection::dgid, EIO, ENOMEM, free_iob(), ib_cmrc_close(), ib_create_conn(), ib_post_send(), ib_cmrc_connection::ibdev, iob_disown, iob_len(), NULL, ib_cmrc_connection::qp, ib_cmrc_connection::service_id, and strerror().

00273                                                                             {
00274         struct ib_cmrc_connection *cmrc =
00275                 container_of ( xfer, struct ib_cmrc_connection, xfer );
00276         int rc;
00277 
00278         /* If no connection has yet been attempted, send this datagram
00279          * as the CM REQ private data.  Otherwise, send it via the QP.
00280          */
00281         if ( ! cmrc->connected ) {
00282 
00283                 /* Abort if we have already sent a CM connection request */
00284                 if ( cmrc->conn ) {
00285                         DBGC ( cmrc, "CMRC %p attempt to send before "
00286                                "connection is complete\n", cmrc );
00287                         rc = -EIO;
00288                         goto out;
00289                 }
00290 
00291                 /* Send via CM connection request */
00292                 cmrc->conn = ib_create_conn ( cmrc->ibdev, cmrc->qp,
00293                                               &cmrc->dgid, &cmrc->service_id,
00294                                               iobuf->data, iob_len ( iobuf ),
00295                                               &ib_cmrc_conn_op );
00296                 if ( ! cmrc->conn ) {
00297                         DBGC ( cmrc, "CMRC %p could not connect\n", cmrc );
00298                         rc = -ENOMEM;
00299                         goto out;
00300                 }
00301 
00302         } else {
00303 
00304                 /* Send via QP */
00305                 if ( ( rc = ib_post_send ( cmrc->ibdev, cmrc->qp, NULL,
00306                                            iob_disown ( iobuf ) ) ) != 0 ) {
00307                         DBGC ( cmrc, "CMRC %p could not send: %s\n",
00308                                cmrc, strerror ( rc ) );
00309                         goto out;
00310                 }
00311 
00312         }
00313         return 0;
00314 
00315  out:
00316         /* Free the I/O buffer if necessary */
00317         free_iob ( iobuf );
00318 
00319         /* Close the connection on any errors */
00320         if ( rc != 0 )
00321                 ib_cmrc_close ( cmrc, rc );
00322 
00323         return rc;
00324 }

static size_t ib_cmrc_xfer_window ( struct xfer_interface *  xfer  )  [static]

Check CMRC flow control window.

Parameters:
xfer Data transfer interface
Return values:
len Length of window

Definition at line 332 of file ib_cmrc.c.

References ib_cmrc_connection::connected, container_of, and IB_MAX_PAYLOAD_SIZE.

00332                                                                   {
00333         struct ib_cmrc_connection *cmrc =
00334                 container_of ( xfer, struct ib_cmrc_connection, xfer );
00335 
00336         /* We indicate a window only when we are successfully
00337          * connected.
00338          */
00339         return ( cmrc->connected ? IB_MAX_PAYLOAD_SIZE : 0 );
00340 }

static void ib_cmrc_xfer_close ( struct xfer_interface *  xfer,
int  rc 
) [static]

Close CMRC data-transfer interface.

Parameters:
xfer Data transfer interface
rc Reason for close

Definition at line 348 of file ib_cmrc.c.

References container_of, DBGC, ib_cmrc_close(), and strerror().

00348                                                                        {
00349         struct ib_cmrc_connection *cmrc =
00350                 container_of ( xfer, struct ib_cmrc_connection, xfer );
00351 
00352         DBGC ( cmrc, "CMRC %p closed: %s\n", cmrc, strerror ( rc ) );
00353         ib_cmrc_close ( cmrc, rc );
00354 }

int ib_cmrc_open ( struct xfer_interface *  xfer,
struct ib_device *  ibdev,
struct ib_gid *  dgid,
struct ib_gid_half *  service_id 
)

Open CMRC connection.

Parameters:
xfer Data transfer interface
ibdev Infiniband device
dgid Destination GID
service_id Service ID
Return values:
rc Returns status code

Definition at line 375 of file ib_cmrc.c.

References ib_cmrc_connection::cq, DBGC, ib_cmrc_connection::dgid, ENOMEM, ib_close(), IB_CMRC_NUM_CQES, IB_CMRC_NUM_RECV_WQES, IB_CMRC_NUM_SEND_WQES, ib_cmrc_shutdown(), ib_create_cq(), ib_create_qp(), ib_destroy_cq(), ib_destroy_qp(), ib_open(), ib_qp_set_ownerdata(), IB_QPT_RC, ib_cmrc_connection::ibdev, memcpy, process_init_stopped(), ib_cmrc_connection::qp, ib_queue_pair::qpn, ref_put(), ib_cmrc_connection::refcnt, ib_cmrc_connection::service_id, ib_cmrc_connection::shutdown, strerror(), ib_cmrc_connection::xfer, xfer_init(), and zalloc().

Referenced by ib_srp_connect().

00376                                                                          {
00377         struct ib_cmrc_connection *cmrc;
00378         int rc;
00379 
00380         /* Allocate and initialise structure */
00381         cmrc = zalloc ( sizeof ( *cmrc ) );
00382         if ( ! cmrc ) {
00383                 rc = -ENOMEM;
00384                 goto err_alloc;
00385         }
00386         xfer_init ( &cmrc->xfer, &ib_cmrc_xfer_operations, &cmrc->refcnt );
00387         cmrc->ibdev = ibdev;
00388         memcpy ( &cmrc->dgid, dgid, sizeof ( cmrc->dgid ) );
00389         memcpy ( &cmrc->service_id, service_id, sizeof ( cmrc->service_id ) );
00390         process_init_stopped ( &cmrc->shutdown, ib_cmrc_shutdown,
00391                                &cmrc->refcnt );
00392 
00393         /* Open Infiniband device */
00394         if ( ( rc = ib_open ( ibdev ) ) != 0 ) {
00395                 DBGC ( cmrc, "CMRC %p could not open device: %s\n",
00396                        cmrc, strerror ( rc ) );
00397                 goto err_open;
00398         }
00399 
00400         /* Create completion queue */
00401         cmrc->cq = ib_create_cq ( ibdev, IB_CMRC_NUM_CQES,
00402                                   &ib_cmrc_completion_ops );
00403         if ( ! cmrc->cq ) {
00404                 DBGC ( cmrc, "CMRC %p could not create completion queue\n",
00405                        cmrc );
00406                 rc = -ENOMEM;
00407                 goto err_create_cq;
00408         }
00409 
00410         /* Create queue pair */
00411         cmrc->qp = ib_create_qp ( ibdev, IB_QPT_RC, IB_CMRC_NUM_SEND_WQES,
00412                                   cmrc->cq, IB_CMRC_NUM_RECV_WQES, cmrc->cq );
00413         if ( ! cmrc->qp ) {
00414                 DBGC ( cmrc, "CMRC %p could not create queue pair\n", cmrc );
00415                 rc = -ENOMEM;
00416                 goto err_create_qp;
00417         }
00418         ib_qp_set_ownerdata ( cmrc->qp, cmrc );
00419         DBGC ( cmrc, "CMRC %p using QPN %lx\n", cmrc, cmrc->qp->qpn );
00420 
00421         /* Attach to parent interface, transfer reference (implicitly)
00422          * to our shutdown process, and return.
00423          */
00424         xfer_plug_plug ( &cmrc->xfer, xfer );
00425         return 0;
00426 
00427         ib_destroy_qp ( ibdev, cmrc->qp );
00428  err_create_qp:
00429         ib_destroy_cq ( ibdev, cmrc->cq );
00430  err_create_cq:
00431         ib_close ( ibdev );
00432  err_open:
00433         ref_put ( &cmrc->refcnt );
00434  err_alloc:
00435         return rc;
00436 }

Variable Documentation

struct ib_connection_operations ib_cmrc_conn_op [static]

Initial value:

 {
        .changed = ib_cmrc_changed,
}
CMRC connection operations.

Definition at line 192 of file ib_cmrc.c.

struct ib_completion_queue_operations ib_cmrc_completion_ops [static]

Initial value:

 {
        .complete_send = ib_cmrc_complete_send,
        .complete_recv = ib_cmrc_complete_recv,
}
Infiniband CMRC completion operations.

Definition at line 258 of file ib_cmrc.c.

struct xfer_interface_operations ib_cmrc_xfer_operations [static]

Initial value:

 {
        .close          = ib_cmrc_xfer_close,
        .vredirect      = ignore_xfer_vredirect,
        .window         = ib_cmrc_xfer_window,
        .alloc_iob      = default_xfer_alloc_iob,
        .deliver_iob    = ib_cmrc_xfer_deliver_iob,
        .deliver_raw    = xfer_deliver_as_iob,
}
CMRC data transfer interface operations.

Definition at line 357 of file ib_cmrc.c.

Generated on Tue Apr 6 20:01:55 2010 for gPXE by  doxygen 1.5.7.1