Commit c876ecb9 authored by Phoebe Buckheister's avatar Phoebe Buckheister 🐍

client: don't retry communication on EFAULT in IB

when reading/writing from a "large" buffer (i.e. one that bypasses the
client caches), the IB sockets can experience EFAULT conditions if
the buffer is shorter than the buffer length provided by the user. is
this happens, do not treat the fault as a fixable error and immediatly
propagate the EFAULT to userspace.

see #520

(cherry picked from commit 1d9177c51338b54d4f93f21dc03f5b717096d395)
parent 96143606
......@@ -340,7 +340,7 @@ err_invalidateSock:
err_fault:
_this->errState = -1;
return -1;
return -EFAULT;
}
......
......@@ -530,6 +530,10 @@ static void __commkit_socketinvalidate_generic(CommKitContext* context,
Logger_logErrFormatted(context->log, context->ops->logContext,
"Communication with inactive node. Node: %s", Node_getNodeIDWithTypeStr(info->node) );
}
else if (info->nodeResult == -FhgfsOpsErr_ADDRESSFAULT)
{
// not a commkit error. release all resources and treat this CTI as done during cleanup.
}
else
{ // "normal" connection error
info->nodeResult = -FhgfsOpsErr_COMMUNICATION;
......@@ -1003,8 +1007,8 @@ static ssize_t __commkit_readfile_receive(CommKitContext* context, ReadfileState
if(unlikely(recvRes < 0) )
{
Logger_logFormatted(context->log, Log_SPAM, context->ops->logContext,
"Request details: receive from %s: %lld bytes",
Node_getNodeIDWithTypeStr(currentState->base.node), (long long)length);
"Request details: receive from %s: %lld bytes (error %zi)",
Node_getNodeIDWithTypeStr(currentState->base.node), (long long)length, recvRes);
}
return recvRes;
......
#include <stdio.h>
#include <time.h>
#include <unistd.h>
#include <errno.h>
#include <sys/mman.h>
int main()
{
time_t startedAt = time(0);
void* buffer = mmap(0, 4096, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (buffer == MAP_FAILED)
return 10;
if (read(0, buffer, 1024 * 1024 * 128) >= 0)
return 1;
if (errno != EFAULT)
return 2;
if (write(1, buffer + sizeof(buffer), 1024 * 1024 * 128) >= 0)
return 3;
if (errno != EFAULT)
return 4;
// retries will eventually fall back to TCP and cause EFAULT, but those will take substantially
// longer than 60 seconds.
if (time(0) - startedAt > 60)
return 5;
return 0;
}
# this test required infiniband. all servers on one node are fine if IB connections exist.
mount = ARGV[0]
raise "need a mountpoint" unless mount
test = caller_locations(0, 1)[0].path.gsub /.pq$/, ".c"
on node do; cd mount do; tempdir do
upload local: test
shell "gcc -std=gnu99 #{test.split("/").last}"
shell "truncate -s#{128 * 1024 * 1024} test"
shell "./a.out <>test"
end; end; end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment