Index of /public/repo/mlnx_ofed/latest/SRPMS (mellanox.com)
目次
orig_os_api
OS内のsocket APIをorig_os_apiに登録し、フックした各APIから呼び出す。
#define GET_ORIG_FUNC(__name) \
if (!orig_os_api.__name) { \
dlerror(); \
assign_dlsym(orig_os_api.__name, #__name); \
const char *fcntlstr = "fcntl64"; \
char *dlerror_str = dlerror(); \
if (!orig_os_api.__name || dlerror_str) { \
if (strcmp(fcntlstr, #__name) != 0) { \
__log_warn("dlsym returned with error '%s' when looking for '%s'", \
(!dlerror_str ? "" : dlerror_str), #__name); \
} else { \
__log_dbg("dlsym returned with error '%s' when looking for '%s'", \
(!dlerror_str ? "" : dlerror_str), #__name); \
} \
} else { \
__log_dbg("dlsym found %p for '%s()'", orig_os_api.__name, #__name); \
} \
}
初期化
//-----------------------------------------------------------------------------
// library init function
//-----------------------------------------------------------------------------
// __attribute__((constructor)) causes the function to be called when
// library is firsrt loaded
// extern "C" int __attribute__((constructor)) sock_redirect_lib_load_constructor(void)
extern "C" int main_init(void)
{
get_orig_funcs();
safe_mce_sys();
g_init_global_ctors_done = false;
vlog_start(PRODUCT_NAME, safe_mce_sys().log_level, safe_mce_sys().log_filename,
safe_mce_sys().log_details, safe_mce_sys().log_colors);
print_vma_global_settings();
check_debug();
check_cpu_speed();
check_locked_mem();
check_netperf_flags();
if (*safe_mce_sys().stats_filename) {
if (check_if_regular_file(safe_mce_sys().stats_filename)) {
vlog_printf(VLOG_WARNING,
"FAILED to create " PRODUCT_NAME
" statistics file. %s is not a regular file.\n",
safe_mce_sys().stats_filename);
} else if (!(g_stats_file = fopen(safe_mce_sys().stats_filename, "w"))) {
vlog_printf(VLOG_WARNING, " Couldn't open statistics file: %s\n",
safe_mce_sys().stats_filename);
}
}
safe_mce_sys().stats_file = g_stats_file;
sock_redirect_main();
return 0;
}Write
socket APIはsock-redirect.cppで、APIごとに定義される。各APIでは、p_socket_objectがあれば、特化パスに入り、そうでなければ通常のsocket APIで処理する。
/* Write IOCNT blocks from IOVEC to FD. Return the number written, or -1.
This function is a cancellation point and therefore not marked with
__THROW. */
extern "C" EXPORT_SYMBOL ssize_t writev(int __fd, const struct iovec *iov, int iovcnt)
{
srdr_logfuncall_entry("fd=%d, %d iov blocks", __fd, iovcnt);
socket_fd_api *p_socket_object = NULL;
p_socket_object = fd_collection_get_sockfd(__fd);
if (p_socket_object) {
vma_tx_call_attr_t tx_arg;
tx_arg.opcode = TX_WRITEV;
tx_arg.attr.msg.iov = (struct iovec *)iov;
tx_arg.attr.msg.sz_iov = iovcnt;
return p_socket_object->tx(tx_arg);
}
BULLSEYE_EXCLUDE_BLOCK_START
if (!orig_os_api.writev) {
get_orig_funcs();
}
BULLSEYE_EXCLUDE_BLOCK_END
return orig_os_api.writev(__fd, iov, iovcnt);
}ssize_t sockinfo_tcp::tx(vma_tx_call_attr_t &tx_arg)
{
return m_ops->tx(tx_arg);
}ssize_t sockinfo_tcp::tcp_tx(vma_tx_call_attr_t &tx_arg)
{
iovec *p_iov = tx_arg.attr.msg.iov;
ssize_t sz_iov = tx_arg.attr.msg.sz_iov;
struct sockaddr *__dst = tx_arg.attr.msg.addr;
socklen_t __dstlen = tx_arg.attr.msg.len;
int __flags = tx_arg.attr.msg.flags;
int errno_tmp = errno;
int total_tx = 0;
unsigned tx_size;
unsigned pos = 0;
int ret = 0;
int poll_count = 0;
uint16_t apiflags = 0;
err_t err;
bool is_dummy = false;
bool block_this_run = false;
bool is_send_zerocopy = false;
void *tx_ptr = NULL;
__off64_t file_offset = 0;
struct xlio_pd_key *pd_key_array = NULL;
/* Let allow OS to process all invalid scenarios to avoid any
* inconsistencies in setting errno values
*/
if (unlikely((m_sock_offload != TCP_SOCK_LWIP) || (NULL == p_iov) || (0 >= sz_iov) ||
(NULL == p_iov[0].iov_base))) {
goto tx_packet_to_os;
}
#ifdef VMA_TIME_MEASURE
TAKE_T_TX_START;
#endif
retry_is_ready:
if (unlikely(!is_rts())) {
if (m_conn_state == TCP_CONN_CONNECTING) {
si_tcp_logdbg("TX while async-connect on socket go to poll");
rx_wait_helper(poll_count, false);
if (m_conn_state == TCP_CONN_CONNECTED) {
goto retry_is_ready;
}
si_tcp_logdbg("TX while async-connect on socket return EAGAIN");
errno = EAGAIN;
} else if (m_conn_state == TCP_CONN_RESETED) {
si_tcp_logdbg("TX on reseted socket");
errno = ECONNRESET;
} else if (m_conn_state == TCP_CONN_ERROR) {
si_tcp_logdbg("TX on connection failed socket");
errno = ECONNREFUSED;
} else {
si_tcp_logdbg("TX on disconnected socket");
errno = EPIPE;
}
#ifdef VMA_TIME_MEASURE
INC_ERR_TX_COUNT;
#endif
return -1;
}
si_tcp_logfunc("tx: iov=%p niovs=%d", p_iov, sz_iov);
if (unlikely(m_sysvar_rx_poll_on_tx_tcp)) {
rx_wait_helper(poll_count, false);
}
lock_tcp_con();
is_dummy = IS_DUMMY_PACKET(__flags);
block_this_run = BLOCK_THIS_RUN(m_b_blocking, __flags);
if (unlikely(is_dummy)) {
apiflags |= VMA_TX_PACKET_DUMMY;
if (!check_dummy_send_conditions(__flags, p_iov, sz_iov)) {
unlock_tcp_con();
errno = EAGAIN;
return -1;
}
}
if (tx_arg.opcode == TX_FILE) {
/*
* TX_FILE is a special operation which reads a single file.
* Each p_iov item contains pointer to file offset and size
* to be read. Pointer to the file descriptor is passed via
* tx_arg.priv.
*/
apiflags |= VMA_TX_FILE;
}
if (!block_this_run && (tx_arg.vma_flags & TX_FLAG_NO_PARTIAL_WRITE)) {
tx_size = 0;
for (int i = 0; i < sz_iov; ++i) {
tx_size += p_iov[i].iov_len;
}
if (unlikely(tcp_sndbuf(&m_pcb) < tx_size)) {
unlock_tcp_con();
errno = EAGAIN;
return -1;
}
}
#ifdef DEFINED_TCP_TX_WND_AVAILABILITY
if (!tcp_is_wnd_available(&m_pcb, p_iov[0].iov_len)) {
unlock_tcp_con();
errno = EAGAIN;
return -1;
}
#endif
/* To force zcopy flow there are two possible ways
* - send() MSG_ZEROCOPY flag should be passed by user application
* and SO_ZEROCOPY activated
* - sendfile() MSG_SEROCOPY flag set internally with opcode TX_FILE
*/
if ((__flags & MSG_ZEROCOPY) && ((m_b_zc) || (tx_arg.opcode == TX_FILE))) {
apiflags |= VMA_TX_PACKET_ZEROCOPY;
is_send_zerocopy = tx_arg.opcode != TX_FILE;
pd_key_array =
(tx_arg.priv.attr == PBUF_DESC_MKEY ? (struct xlio_pd_key *)tx_arg.priv.map : NULL);
}
for (int i = 0; i < sz_iov; i++) {
si_tcp_logfunc("iov:%d base=%p len=%d", i, p_iov[i].iov_base, p_iov[i].iov_len);
pos = 0;
if ((tx_arg.opcode == TX_FILE) && !(apiflags & VMA_TX_PACKET_ZEROCOPY)) {
file_offset = *(__off64_t *)p_iov[i].iov_base;
tx_ptr = &file_offset;
} else {
tx_ptr = p_iov[i].iov_base;
if ((tx_arg.priv.attr == PBUF_DESC_MKEY) && pd_key_array) {
tx_arg.priv.mkey = pd_key_array[i].mkey;
}
}
while (pos < p_iov[i].iov_len) {
tx_size = tcp_sndbuf(&m_pcb);
/* Process a case when space is not available at the sending socket
* to hold the message to be transmitted
* Nonblocking socket:
* - no data is buffered: return (-1) and EAGAIN
* - some data is buffered: return number of bytes ready to be sent
* Blocking socket:
* - block until space is available
*/
if (tx_size == 0) {
if (unlikely(!is_rts())) {
si_tcp_logdbg("TX on disconnected socket");
ret = -1;
errno = ECONNRESET;
goto err;
}
// force out TCP data before going on wait()
tcp_output(&m_pcb);
/* Set return values for nonblocking socket and finish processing */
if (!block_this_run) {
// non blocking socket should return inorder not to tx_wait()
if (total_tx > 0) {
m_tx_consecutive_eagain_count = 0;
goto done;
} else {
m_tx_consecutive_eagain_count++;
if (m_tx_consecutive_eagain_count >= TX_CONSECUTIVE_EAGAIN_THREASHOLD) {
// in case of zero sndbuf and non-blocking just try once polling CQ for
// ACK
rx_wait(poll_count, false);
m_tx_consecutive_eagain_count = 0;
}
ret = -1;
errno = EAGAIN;
goto err;
}
}
tx_size = tx_wait(ret, true);
}
if (tx_size > p_iov[i].iov_len - pos) {
tx_size = p_iov[i].iov_len - pos;
}
if (is_send_zerocopy) {
/*
* For send zerocopy we don't support pbufs which
* cross huge page boundaries. To avoid forming
* such a pbuf, we have to adjust tx_size, so
* tcp_write receives a buffer which doesn't cross
* the boundary.
*/
unsigned remainder =
~m_user_huge_page_mask + 1 - ((uint64_t)tx_ptr & ~m_user_huge_page_mask);
if (tx_size > remainder) {
tx_size = remainder;
}
}
retry_write:
if (unlikely(!is_rts())) {
si_tcp_logdbg("TX on disconnected socket");
ret = -1;
errno = ECONNRESET;
goto err;
}
if (unlikely(g_b_exit)) {
if (total_tx > 0) {
goto done;
} else {
ret = -1;
errno = EINTR;
si_tcp_logdbg("returning with: EINTR");
goto err;
}
}
err = tcp_write(&m_pcb, tx_ptr, tx_size, apiflags, &tx_arg.priv);
if (unlikely(err != ERR_OK)) {
if (unlikely(err == ERR_CONN)) { // happens when remote drops during big write
si_tcp_logdbg("connection closed: tx'ed = %d", total_tx);
shutdown(SHUT_WR);
if (total_tx > 0) {
goto done;
}
errno = EPIPE;
unlock_tcp_con();
#ifdef VMA_TIME_MEASURE
INC_ERR_TX_COUNT;
#endif
return -1;
}
if (unlikely(err != ERR_MEM)) {
// we should not get here...
BULLSEYE_EXCLUDE_BLOCK_START
si_tcp_logpanic("tcp_write return: %d", err);
BULLSEYE_EXCLUDE_BLOCK_END
}
/* Set return values for nonblocking socket and finish processing */
if (!block_this_run) {
if (total_tx > 0) {
goto done;
} else {
ret = -1;
errno = EAGAIN;
goto err;
}
}
rx_wait(poll_count, true);
// AlexV:Avoid from going to sleep, for the blocked socket of course, since
// progress engine may consume an arrived credit and it will not wakeup the
// transmit thread.
poll_count = 0;
goto retry_write;
}
if (tx_arg.opcode == TX_FILE && !(apiflags & VMA_TX_PACKET_ZEROCOPY)) {
file_offset += tx_size;
} else {
tx_ptr = (void *)((char *)tx_ptr + tx_size);
}
pos += tx_size;
total_tx += tx_size;
}
}
done:
tcp_output(&m_pcb); // force data out ★ここで送信？
if (unlikely(is_dummy)) {
m_p_socket_stats->counters.n_tx_dummy++;
} else if (total_tx) {
m_p_socket_stats->counters.n_tx_sent_byte_count += total_tx;
m_p_socket_stats->counters.n_tx_sent_pkt_count++;
m_p_socket_stats->n_tx_ready_byte_count += total_tx;
}
/* Each send call with MSG_ZEROCOPY that successfully sends
* data increments the counter.
* The counter is not incremented on failure or if called with length zero.
*/
if (is_send_zerocopy && (total_tx > 0)) {
if (m_last_zcdesc->tx.zc.id != (uint32_t)atomic_read(&m_zckey)) {
si_tcp_logerr("Invalid tx zcopy operation");
} else {
atomic_fetch_and_inc(&m_zckey);
}
}
unlock_tcp_con();
#ifdef VMA_TIME_MEASURE
TAKE_T_TX_END;
#endif
/* Restore errno on function entry in case success */
errno = errno_tmp;
return total_tx;
err:
#ifdef VMA_TIME_MEASURE
INC_ERR_TX_COUNT;
#endif
// nothing send nb mode or got some other error
if (errno == EAGAIN) {
m_p_socket_stats->counters.n_tx_eagain++;
} else {
m_p_socket_stats->counters.n_tx_errors++;
}
unlock_tcp_con();
return ret;
tx_packet_to_os:
#ifdef VMA_TIME_MEASURE
INC_GO_TO_OS_TX_COUNT;
#endif
ret = socket_fd_api::tx_os(tx_arg.opcode, p_iov, sz_iov, __flags, __dst, __dstlen);
save_stats_tx_os(ret);
return ret;
}/**
* Find out what we can send and send it
*
* @param pcb Protocol control block for the TCP connection to send data
* @return ERR_OK if data has been sent or nothing to send
* another err_t on error
*/
err_t
tcp_output(struct tcp_pcb *pcb)
{
struct tcp_seg *seg, *useg;
u32_t wnd, snd_nxt;
err_t rc;
#if TCP_CWND_DEBUG
s16_t i = 0;
#endif /* TCP_CWND_DEBUG */
/* First, check if we are invoked by the TCP input processing
code. If so, we do not output anything. Instead, we rely on the
input processing code to call us when input processing is done
with. */
if (pcb->is_in_input) {
return ERR_OK;
}
wnd = LWIP_MIN(pcb->snd_wnd, pcb->cwnd);
LWIP_DEBUGF(TCP_CWND_DEBUG, ("tcp_output: snd_wnd %"U32_F", cwnd %"U32_F
", wnd %"U32_F"\n",pcb->snd_wnd, pcb->cwnd, wnd ));
seg = pcb->unsent;
/* If the TF_ACK_NOW flag is set and no data will be sent (either
* because the ->unsent queue is empty or because the window does
* not allow it), construct an empty ACK segment and send it.
*
* If data is to be sent, we will just piggyback the ACK (see below).
*/
if ((pcb->flags & TF_ACK_NOW) &&
(seg == NULL ||
seg->seqno - pcb->lastack + seg->len > wnd)) {
return tcp_send_empty_ack(pcb);
}
#if TCP_OUTPUT_DEBUG
if (seg == NULL) {
LWIP_DEBUGF(TCP_OUTPUT_DEBUG, ("tcp_output: nothing to send (%p)\n",
(void*)pcb->unsent));
}
#endif /* TCP_OUTPUT_DEBUG */
#if TCP_CWND_DEBUG
if (seg == NULL) {
LWIP_DEBUGF(TCP_CWND_DEBUG, ("tcp_output: snd_wnd %"U32_F
", cwnd %"U32_F", wnd %"U32_F
", seg == NULL, ack %"U32_F"\n",
pcb->snd_wnd, pcb->cwnd, wnd, pcb->lastack));
} else {
LWIP_DEBUGF(TCP_CWND_DEBUG,
("tcp_output: snd_wnd %"U32_F", cwnd %"U32_F", wnd %"U32_F
", effwnd %"U32_F", seq %"U32_F", ack %"U32_F"\n",
pcb->snd_wnd, pcb->cwnd, wnd,
ntohl(seg->tcphdr->seqno) - pcb->lastack + seg->len,
ntohl(seg->tcphdr->seqno), pcb->lastack));
}
#endif /* TCP_CWND_DEBUG */
#if TCP_TSO_DEBUG
if (seg) {
LWIP_DEBUGF(TCP_TSO_DEBUG | LWIP_DBG_TRACE,
("tcp_output: wnd: %-5d unsent %s\n",
wnd, _dump_seg(pcb->unsent)));
}
#endif /* TCP_TSO_DEBUG */
while (seg) {
/* TSO segment can be in unsent queue only in case of retransmission.
* Clear TSO flag, tcp_split_segment() and tcp_tso_segment() will handle
* all scenarios further.
*/
seg->flags &= ~TF_SEG_OPTS_TSO;
if (TCP_SEQ_LT(seg->seqno, pcb->snd_nxt) && seg->p && seg->p->len != seg->p->tot_len) {
tcp_split_rexmit(pcb, seg);
}
/* Split the segment in case of a small window */
if ((NULL == pcb->unacked) && (wnd) && ((seg->len + seg->seqno - pcb->lastack) > wnd)) {
LWIP_ASSERT("tcp_output: no window for dummy packet", !LWIP_IS_DUMMY_SEGMENT(seg));
tcp_split_segment(pcb, seg, wnd);
}
/* data available and window allows it to be sent? */
if (((seg->seqno - pcb->lastack + seg->len) <= wnd)){
LWIP_ASSERT("RST not expected here!",
(TCPH_FLAGS(seg->tcphdr) & TCP_RST) == 0);
/* Stop sending if the nagle algorithm would prevent it
* Don't stop:
* - if tcp_write had a memory error before (prevent delayed ACK timeout) or
* - if this is not a dummy segment
* - if FIN was already enqueued for this PCB (SYN is always alone in a segment -
* either seg->next != NULL or pcb->unacked == NULL;
* RST is no sent using tcp_write/tcp_output.
*/
if((tcp_do_output_nagle(pcb) == 0) &&
!LWIP_IS_DUMMY_SEGMENT(seg) &&
((pcb->flags & (TF_NAGLEMEMERR | TF_FIN)) == 0)){
if ( pcb->snd_sml_snt > (pcb->unacked ? pcb->unacked->len : 0) ) {
break;
}
else {
if ( (u32_t)((seg->next ? seg->next->len : 0) + seg->len) <= pcb->snd_sml_add ) {
pcb->snd_sml_snt = pcb->snd_sml_add;
}
}
}
/* Use TSO send operation in case TSO is enabled
* and current segment is not retransmitted
*/
if (tcp_tso(pcb)) {
tcp_tso_segment(pcb, seg, wnd);
}
#if TCP_CWND_DEBUG
LWIP_DEBUGF(TCP_CWND_DEBUG, ("tcp_output: snd_wnd %"U32_F", cwnd %"U16_F", wnd %"U32_F", effwnd %"U32_F", seq %"U32_F", ack %"U32_F", i %"S16_F"\n",
pcb->snd_wnd, pcb->cwnd, wnd,
ntohl(seg->tcphdr->seqno) + seg->len -
pcb->lastack,
ntohl(seg->tcphdr->seqno), pcb->lastack, i));
++i;
#endif /* TCP_CWND_DEBUG */
// Send ack now if the packet is a dummy packet
if (LWIP_IS_DUMMY_SEGMENT(seg) && (pcb->flags & (TF_ACK_DELAY | TF_ACK_NOW))) {
tcp_send_empty_ack(pcb);
}
if (get_tcp_state(pcb) != SYN_SENT) {
TCPH_SET_FLAG(seg->tcphdr, TCP_ACK);
pcb->flags &= ~(TF_ACK_DELAY | TF_ACK_NOW);
}
#if TCP_OVERSIZE_DBGCHECK
seg->oversize_left = 0;
#endif /* TCP_OVERSIZE_DBGCHECK */
rc = tcp_output_segment(seg, pcb);
if (rc != ERR_OK) {
if (rc == ERR_WOULDBLOCK)
break;
return rc;
}
pcb->unsent = seg->next;
snd_nxt = seg->seqno + TCP_TCPLEN(seg);
if (TCP_SEQ_LT(pcb->snd_nxt, snd_nxt) && !LWIP_IS_DUMMY_SEGMENT(seg)) {
pcb->snd_nxt = snd_nxt;
}
/* put segment on unacknowledged list if length > 0 */
if (TCP_TCPLEN(seg) > 0) {
seg->next = NULL;
// unroll dummy segment
if (LWIP_IS_DUMMY_SEGMENT(seg)) {
pcb->snd_lbb -= seg->len;
pcb->snd_buf += seg->len;
pcb->snd_queuelen -= pbuf_clen(seg->p);
tcp_tx_seg_free(pcb, seg);
} else {
/* unacked list is empty? */
if (pcb->unacked == NULL) {
pcb->unacked = seg;
pcb->last_unacked = seg;
/* unacked list is not empty? */
} else {
/* In the case of fast retransmit, the packet should not go to the tail
* of the unacked queue, but rather somewhere before it. We need to check for
* this case. -STJ Jul 27, 2004 */
useg = pcb->last_unacked;
if (TCP_SEQ_LT(seg->seqno, useg->seqno)) {
/* add segment to before tail of unacked list, keeping the list sorted */
struct tcp_seg **cur_seg = &(pcb->unacked);
while (*cur_seg &&
TCP_SEQ_LT((*cur_seg)->seqno, seg->seqno)) {
cur_seg = &((*cur_seg)->next );
}
LWIP_ASSERT("Value of last_unacked is invalid",
*cur_seg != pcb->last_unacked->next);
seg->next = (*cur_seg);
(*cur_seg) = seg;
} else {
/* add segment to tail of unacked list */
useg->next = seg;
pcb->last_unacked = seg;
}
}
}
/* do not queue empty segments on the unacked list */
} else {
tcp_tx_seg_free(pcb, seg);
}
seg = pcb->unsent;
}
else {
break;
}
}
if (pcb->unsent == NULL) {
/* We have sent all pending segments, reset last_unsent */
pcb->last_unsent = NULL;
#if TCP_OVERSIZE
pcb->unsent_oversize = 0;
#endif /* TCP_OVERSIZE */
}
pcb->flags &= ~TF_NAGLEMEMERR;
// Fetch buffers for the next packet.
if (!pcb->seg_alloc) {
// Fetch tcp segment for the next packet.
pcb->seg_alloc = tcp_create_segment(pcb, NULL, 0, 0, 0);
}
if (!pcb->pbuf_alloc) {
// Fetch pbuf for the next packet.
pcb->pbuf_alloc = tcp_tx_pbuf_alloc(pcb, 0, PBUF_RAM, NULL, NULL);
}
return ERR_OK;
}