[c] What can cause a “Resource temporarily unavailable” on sock send() command

What can cause a Resource temporarily unavailable error on a socket send() command? The socket is setup as AF_UNIX, SOCK_STREAM. It works most of the time, but occasionally gets this error. The receiving end of the socket appears to be working properly.

I know this isn't very detailed, but I'm just looking for general ideas. Thanks!

Let'e me give an example:

1. client connect to server, and send 1MB data to server every 1 second.

2. server side accept a connection, and then sleep 20 second, without recv msg from client.So the tcp send buffer in the client side will be full.

Code in client side:

#include <arpa/inet.h>
#include <sys/socket.h>
#include <stdio.h>
#include <errno.h>
#include <fcntl.h>
#include <stdlib.h>
#include <string.h>
#define exit_if(r, ...)                                                                          \
    if (r) {                                                                                     \
        printf(__VA_ARGS__);                                                                     \
        printf("%s:%d error no: %d error msg %s\n", __FILE__, __LINE__, errno, strerror(errno)); \
        exit(1);                                                                                 \
    }

void setNonBlock(int fd) {
    int flags = fcntl(fd, F_GETFL, 0);
    exit_if(flags < 0, "fcntl failed");
    int r = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
    exit_if(r < 0, "fcntl failed");
}

void test_full_sock_buf_1(){
    short port = 8000;
    struct sockaddr_in addr;
    memset(&addr, 0, sizeof addr);
    addr.sin_family = AF_INET;
    addr.sin_port = htons(port);
    addr.sin_addr.s_addr = INADDR_ANY;


    int fd = socket(AF_INET, SOCK_STREAM, 0);
    exit_if(fd<0, "create socket error");

    int ret = connect(fd, (struct sockaddr *) &addr, sizeof(struct sockaddr));
    exit_if(ret<0, "connect to server error");
    setNonBlock(fd);

    printf("connect to server success");

    const int LEN = 1024 * 1000;
    char msg[LEN];  // 1MB data
    memset(msg, 'a', LEN);

    for (int i = 0; i < 1000; ++i) {
        int len = send(fd, msg, LEN, 0);
        printf("send: %d, erron: %d, %s \n", len, errno, strerror(errno));
        sleep(1);
    }

}

int main(){
    test_full_sock_buf_1();

    return 0;
}

Code in server side:

    #include <arpa/inet.h>
    #include <sys/socket.h>
    #include <stdio.h>
    #include <errno.h>
    #include <fcntl.h>
    #include <stdlib.h>
    #include <string.h>
    #define exit_if(r, ...)                                                                          \
        if (r) {                                                                                     \
            printf(__VA_ARGS__);                                                                     \
            printf("%s:%d error no: %d error msg %s\n", __FILE__, __LINE__, errno, strerror(errno)); \
            exit(1);                                                                                 \
        }
void test_full_sock_buf_1(){

    int listenfd = socket(AF_INET, SOCK_STREAM, 0);
    exit_if(listenfd<0, "create socket error");

    short port = 8000;
    struct sockaddr_in addr;
    memset(&addr, 0, sizeof addr);
    addr.sin_family = AF_INET;
    addr.sin_port = htons(port);
    addr.sin_addr.s_addr = INADDR_ANY;

    int r = ::bind(listenfd, (struct sockaddr *) &addr, sizeof(struct sockaddr));
    exit_if(r<0, "bind socket error");

    r = listen(listenfd, 100);
    exit_if(r<0, "listen socket error");

    struct sockaddr_in raddr;
    socklen_t rsz = sizeof(raddr);
    int cfd = accept(listenfd, (struct sockaddr *) &raddr, &rsz);
    exit_if(cfd<0, "accept socket error");

    sockaddr_in peer;
    socklen_t alen = sizeof(peer);
    getpeername(cfd, (sockaddr *) &peer, &alen);

    printf("accept a connection from %s:%d\n", inet_ntoa(peer.sin_addr), ntohs(peer.sin_port));

    printf("but now I will sleep 15 second, then exit");
    sleep(15);
}

Start server side, then start client side.

server side may output:

accept a connection from 127.0.0.1:35764
but now I will sleep 15 second, then exit
Process finished with exit code 0

client side may output:

connect to server successsend: 1024000, erron: 0, Success 
send: 1024000, erron: 0, Success 
send: 1024000, erron: 0, Success 
send: 552190, erron: 0, Success 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 11, Resource temporarily unavailable 
send: -1, erron: 104, Connection reset by peer 
send: -1, erron: 32, Broken pipe 
send: -1, erron: 32, Broken pipe 
send: -1, erron: 32, Broken pipe 
send: -1, erron: 32, Broken pipe 
send: -1, erron: 32, Broken pipe 

You can see, as the server side doesn't recv the data from client, so when the client side tcp buffer get full, but you still send data, so you may get Resource temporarily unavailable error.

That's because you're using a non-blocking socket and the output buffer is full.

From the send() man page

   When the message does not fit into  the  send  buffer  of  the  socket,
   send() normally blocks, unless the socket has been placed in non-block-
   ing I/O mode.  In non-blocking mode it  would  return  EAGAIN  in  this
   case.  

EAGAIN is the error code tied to "Resource temporarily unavailable"

Consider using select() to get a better control of this behaviours