Commit c7403fe8 authored by Bernd Lietzow's avatar Bernd Lietzow 🐨

update to release 7.1.3

parent 308db034
...@@ -103,7 +103,7 @@ endef ...@@ -103,7 +103,7 @@ endef
comm_debug-install: install_name=comm-debug comm_debug-install: install_name=comm-debug
$(foreach D,$(DAEMONS),$(eval $(call install_component,$D,sbin))) $(foreach D,$(DAEMONS),$(eval $(call install_component,$D,sbin)))
$(foreach U,$(filter-out event_listener,$(UTILS)),$(eval $(call install_component,$U,sbin))) $(foreach U,$(filter-out event_listener $(UTILS)),$(eval $(call install_component,$U,sbin)))
.PHONY: daemons-install .PHONY: daemons-install
daemons-install: $(patsubst %,%-install,$(DAEMONS)) daemons-install: $(patsubst %,%-install,$(DAEMONS))
......
...@@ -176,7 +176,7 @@ void App::runNormal() ...@@ -176,7 +176,7 @@ void App::runNormal()
bool foundRdmaInterfaces = NetworkInterfaceCard::checkAndAddRdmaCapability(localNicList); bool foundRdmaInterfaces = NetworkInterfaceCard::checkAndAddRdmaCapability(localNicList);
if (foundRdmaInterfaces) if (foundRdmaInterfaces)
localNicList.sort(&NetworkInterfaceCard::nicAddrPreferenceComp); // re-sort the niclist localNicList.sort(NetworkInterfaceCard::NicAddrComp{&allowedInterfaces}); // re-sort the niclist
} }
logInfos(); logInfos();
...@@ -224,7 +224,6 @@ void App::initLocalNodeInfo() ...@@ -224,7 +224,6 @@ void App::initLocalNodeInfo()
{ {
unsigned portUDP = cfg->getConnAdmonPortUDP(); unsigned portUDP = cfg->getConnAdmonPortUDP();
StringList allowedInterfaces;
std::string interfacesFilename = cfg->getConnInterfacesFile(); std::string interfacesFilename = cfg->getConnInterfacesFile();
if(interfacesFilename.length() ) if(interfacesFilename.length() )
Config::loadStringListFile(interfacesFilename.c_str(), allowedInterfaces); Config::loadStringListFile(interfacesFilename.c_str(), allowedInterfaces);
...@@ -234,7 +233,7 @@ void App::initLocalNodeInfo() ...@@ -234,7 +233,7 @@ void App::initLocalNodeInfo()
if(localNicList.empty() ) if(localNicList.empty() )
throw InvalidConfigException("Couldn't find any usable NIC"); throw InvalidConfigException("Couldn't find any usable NIC");
localNicList.sort(&NetworkInterfaceCard::nicAddrPreferenceComp); localNicList.sort(NetworkInterfaceCard::NicAddrComp{&allowedInterfaces});
std::string nodeID = System::getHostname(); std::string nodeID = System::getHostname();
......
...@@ -72,6 +72,7 @@ class App : public AbstractApp ...@@ -72,6 +72,7 @@ class App : public AbstractApp
Config* cfg; Config* cfg;
RuntimeConfig *runtimeCfg; RuntimeConfig *runtimeCfg;
LogContext* log; LogContext* log;
std::list<std::string> allowedInterfaces;
LockFD pidFileLockFD; LockFD pidFileLockFD;
......
...@@ -291,6 +291,7 @@ $(call define_if_matches, KERNEL_HAS_ALLOC_WORKQUEUE, "alloc_workqueue", workque ...@@ -291,6 +291,7 @@ $(call define_if_matches, KERNEL_HAS_ALLOC_WORKQUEUE, "alloc_workqueue", workque
$(call define_if_matches, KERNEL_HAS_WQ_RESCUER, "WQ_RESCUER", workqueue.h) $(call define_if_matches, KERNEL_HAS_WQ_RESCUER, "WQ_RESCUER", workqueue.h)
$(call define_if_matches, KERNEL_HAS_WAIT_QUEUE_ENTRY_T, "wait_queue_entry_t", wait.h) $(call define_if_matches, KERNEL_HAS_WAIT_QUEUE_ENTRY_T, "wait_queue_entry_t", wait.h)
$(call define_if_matches, KERNEL_HAS_CURRENT_FS_TIME, "current_fs_time", fs.h) $(call define_if_matches, KERNEL_HAS_CURRENT_FS_TIME, "current_fs_time", fs.h)
$(call define_if_matches, KERNEL_HAS_64BIT_TIMESTAMPS, "struct timespec64[[:space:]]\+i_atime;", fs.h)
# inodeChangeRes was changed to setattr_prepare in vanilla 4.9 # inodeChangeRes was changed to setattr_prepare in vanilla 4.9
$(call define_if_matches, KERNEL_HAS_SETATTR_PREPARE, "int setattr_prepare", fs.h) $(call define_if_matches, KERNEL_HAS_SETATTR_PREPARE, "int setattr_prepare", fs.h)
......
...@@ -101,7 +101,12 @@ BEEGFS_CFLAGS += -DBEEGFS_OPENTK_IBVERBS ...@@ -101,7 +101,12 @@ BEEGFS_CFLAGS += -DBEEGFS_OPENTK_IBVERBS
endif endif
ifneq ($(OFED_INCLUDE_PATH),) ifneq ($(OFED_INCLUDE_PATH),)
BEEGFS_CFLAGS += -I$(OFED_INCLUDE_PATH) BEEGFS_CFLAGS += -I$(OFED_INCLUDE_PATH)
module: $(OFED_INCLUDE_PATH)/rdma/rdma_cm.h
$(OFED_INCLUDE_PATH)/rdma/rdma_cm.h:
$(error OFED_INCLUDE_PATH not valid: $(OFED_INCLUDE_PATH))
endif endif
# OFED API version # OFED API version
......
...@@ -94,6 +94,10 @@ endif ...@@ -94,6 +94,10 @@ endif
ifneq ($(OFED_INCLUDE_PATH),) ifneq ($(OFED_INCLUDE_PATH),)
BEEGFS_CFLAGS += -I$(OFED_INCLUDE_PATH) BEEGFS_CFLAGS += -I$(OFED_INCLUDE_PATH)
module: $(OFED_INCLUDE_PATH)/rdma/rdma_cm.h
$(OFED_INCLUDE_PATH)/rdma/rdma_cm.h:
$(error OFED_INCLUDE_PATH not valid: $(OFED_INCLUDE_PATH))
endif endif
# OFED API version # OFED API version
......
...@@ -513,7 +513,7 @@ bool __App_initLocalNodeInfo(App* this) ...@@ -513,7 +513,7 @@ bool __App_initLocalNodeInfo(App* this)
} }
// created sorted nicList clone // created sorted nicList clone
ListTk_cloneSortNicAddressList(&nicList, &sortedNicList); ListTk_cloneSortNicAddressList(&nicList, &sortedNicList, &this->allowedInterfaces);
// prepare clientID and create localNode // prepare clientID and create localNode
......
...@@ -151,7 +151,13 @@ ...@@ -151,7 +151,13 @@
typedef wait_queue_entry_t wait_queue_t; typedef wait_queue_entry_t wait_queue_t;
#endif #endif
#if !defined(KERNEL_HAS_CURRENT_FS_TIME) #if defined(KERNEL_HAS_64BIT_TIMESTAMPS)
static inline struct timespec64 current_fs_time(struct super_block *sb)
{
struct timespec64 now = current_kernel_time64();
return timespec64_trunc(now, sb->s_time_gran);
}
#elif !defined(KERNEL_HAS_CURRENT_FS_TIME)
static inline struct timespec current_fs_time(struct super_block *sb) static inline struct timespec current_fs_time(struct super_block *sb)
{ {
struct timespec now = current_kernel_time(); struct timespec now = current_kernel_time();
......
...@@ -38,27 +38,3 @@ bool NicAddress_preferenceComp(const NicAddress* lhs, const NicAddress* rhs) ...@@ -38,27 +38,3 @@ bool NicAddress_preferenceComp(const NicAddress* lhs, const NicAddress* rhs)
// this is the original IP-order version // this is the original IP-order version
return lhsHostOrderIP > rhsHostOrderIP; return lhsHostOrderIP > rhsHostOrderIP;
} }
/**
* Compares elements suitable for a tree.
*
* @param key1 NicAddress pointer
* @param key2 NicAddress pointer
* @return: <0 if key1<key2, 0 for equal keys, >0 otherwise
*/
int NicAddress_treeComparator(const void* key1, const void* key2)
{
NicAddress* lhs = (NicAddress*)key1;
NicAddress* rhs = (NicAddress*)key2;
// Note: We interpret "preferred" elements as "smaller" elements here.
if(NicAddress_preferenceComp(lhs, rhs) )
return -1;
if(NicAddress_preferenceComp(rhs, lhs) )
return 1;
return 0;
}
...@@ -19,7 +19,6 @@ typedef struct NicListCapabilities NicListCapabilities; ...@@ -19,7 +19,6 @@ typedef struct NicListCapabilities NicListCapabilities;
extern bool NicAddress_preferenceComp(const NicAddress* lhs, const NicAddress* rhs); extern bool NicAddress_preferenceComp(const NicAddress* lhs, const NicAddress* rhs);
extern int NicAddress_treeComparator(const void* key1, const void* key2);
// inliners // inliners
static inline void NicAddress_ipToStr(struct in_addr ipAddr, char* outStr); static inline void NicAddress_ipToStr(struct in_addr ipAddr, char* outStr);
......
...@@ -36,6 +36,24 @@ ...@@ -36,6 +36,24 @@
__func__, __LINE__, ##__VA_ARGS__) __func__, __LINE__, ##__VA_ARGS__)
/* 4.19 added const qualifiers to ib_post_send and ib_post_recv. */
typedef __typeof__(
__builtin_choose_expr(
__builtin_types_compatible_p(
__typeof__(&ib_post_send),
int (*)(struct ib_qp*, struct ib_send_wr*, struct ib_send_wr**)),
(struct ib_send_wr*) 0,
(const struct ib_send_wr*) 0))
_bad_send_wr;
typedef __typeof__(
__builtin_choose_expr(
__builtin_types_compatible_p(
__typeof__(&ib_post_recv),
int (*)(struct ib_qp*, struct ib_recv_wr*, struct ib_recv_wr**)),
(struct ib_recv_wr*) 0,
(const struct ib_recv_wr*) 0))
_bad_recv_wr;
bool IBVSocket_init(IBVSocket* _this) bool IBVSocket_init(IBVSocket* _this)
{ {
...@@ -810,7 +828,7 @@ err_cleanup: ...@@ -810,7 +828,7 @@ err_cleanup:
int __IBVSocket_postRecv(IBVSocket* _this, IBVCommContext* commContext, size_t bufIndex) int __IBVSocket_postRecv(IBVSocket* _this, IBVCommContext* commContext, size_t bufIndex)
{ {
struct ib_recv_wr wr; struct ib_recv_wr wr;
struct ib_recv_wr* bad_wr; _bad_recv_wr bad_wr;
int postRes; int postRes;
commContext->sendBufs[bufIndex].lists[0].length = commContext->commCfg.bufSize; commContext->sendBufs[bufIndex].lists[0].length = commContext->commCfg.bufSize;
...@@ -843,7 +861,7 @@ int IBVSocket_checkConnection(IBVSocket* _this) ...@@ -843,7 +861,7 @@ int IBVSocket_checkConnection(IBVSocket* _this)
# define wr_of(wr) (wr) # define wr_of(wr) (wr)
struct ib_send_wr wr; struct ib_send_wr wr;
#endif #endif
struct ib_send_wr *bad_wr; _bad_send_wr bad_wr;
int postRes; int postRes;
int waitRes; int waitRes;
...@@ -895,7 +913,7 @@ int __IBVSocket_postSend(IBVSocket* _this, size_t bufIndex) ...@@ -895,7 +913,7 @@ int __IBVSocket_postSend(IBVSocket* _this, size_t bufIndex)
{ {
IBVCommContext* commContext = _this->commContext; IBVCommContext* commContext = _this->commContext;
struct ib_send_wr wr; struct ib_send_wr wr;
struct ib_send_wr *bad_wr; _bad_send_wr bad_wr;
int postRes; int postRes;
wr.wr_id = bufIndex; wr.wr_id = bufIndex;
...@@ -1810,7 +1828,7 @@ struct ib_cq* __IBVSocket_createCompletionQueue(struct ib_device* device, ...@@ -1810,7 +1828,7 @@ struct ib_cq* __IBVSocket_createCompletionQueue(struct ib_device* device,
{ {
#if defined (BEEGFS_OFED_1_2_API) && BEEGFS_OFED_1_2_API >= 1 #if defined (BEEGFS_OFED_1_2_API) && BEEGFS_OFED_1_2_API >= 1
return ib_create_cq(device, comp_handler, event_handler, cq_context, cqe); return ib_create_cq(device, comp_handler, event_handler, cq_context, cqe);
#elif defined OFED_HAS_IB_CREATE_CQATTR #elif defined OFED_HAS_IB_CREATE_CQATTR || defined ib_create_cq
struct ib_cq_init_attr attrs = { struct ib_cq_init_attr attrs = {
.cqe = cqe, .cqe = cqe,
.comp_vector = 0, .comp_vector = 0,
......
...@@ -6,6 +6,41 @@ ...@@ -6,6 +6,41 @@
#include <common/toolkit/tree/PointerRBTreeIter.h> #include <common/toolkit/tree/PointerRBTreeIter.h>
#include "ListTk.h" #include "ListTk.h"
#include <linux/sort.h>
struct nicaddr_sort_entry {
StrCpyList* preferences;
NicAddress* addr;
};
static int nicaddr_sort_comp(const void* l, const void* r)
{
const struct nicaddr_sort_entry* lhs = l;
const struct nicaddr_sort_entry* rhs = r;
StrCpyListIter it;
StrCpyListIter_init(&it, lhs->preferences);
while (!StrCpyListIter_end(&it)) {
const char* value = StrCpyListIter_value(&it);
if (strcmp(value, lhs->addr->name) == 0)
return -1;
if (strcmp(value, rhs->addr->name) == 0)
return 1;
StrCpyListIter_next(&it);
}
if (NicAddress_preferenceComp(lhs->addr, rhs->addr))
return -1;
if (NicAddress_preferenceComp(rhs->addr, lhs->addr))
return 1;
return 0;
}
void ListTk_cloneNicAddressList(NicAddressList* nicList, NicAddressList* nicListClone) void ListTk_cloneNicAddressList(NicAddressList* nicList, NicAddressList* nicListClone)
{ {
...@@ -35,51 +70,44 @@ void ListTk_cloneNicAddressList(NicAddressList* nicList, NicAddressList* nicList ...@@ -35,51 +70,44 @@ void ListTk_cloneNicAddressList(NicAddressList* nicList, NicAddressList* nicList
/** /**
* Returns a sorted clone of a nicList. * Returns a sorted clone of a nicList.
* *
* Note: As this is based on a tree insertion, it won't work when there are duplicate elements * Note: the comparison implemented here is a valid only if preferences contains
* in the list. * all possible names ever encountered, or none at all.
*/ */
void ListTk_cloneSortNicAddressList(NicAddressList* nicList, NicAddressList* nicListClone) void ListTk_cloneSortNicAddressList(NicAddressList* nicList, NicAddressList* nicListClone,
StrCpyList* preferences)
{ {
NicAddressListIter listIter; NicAddressListIter listIter;
RBTree tree;
RBTreeIter treeIter;
PointerRBTree_init(&tree, NicAddress_treeComparator); struct nicaddr_sort_entry* list, *p;
// sort => insert each list elem into the tree (no copying) /* i'm so sorry. we can't indicate failure here without a lot of work in App, and the
* lists are usually tiny anyway. */
list = kcalloc(NicAddressList_length(nicList), sizeof(*list), GFP_NOFS | __GFP_NOFAIL);
p = list;
NicAddressListIter_init(&listIter, nicList); NicAddressListIter_init(&listIter, nicList);
for( ; !NicAddressListIter_end(&listIter); NicAddressListIter_next(&listIter) ) while (!NicAddressListIter_end(&listIter)) {
{ p->preferences = preferences;
NicAddress* nicAddr = NicAddressListIter_value(&listIter); p->addr = NicAddressListIter_value(&listIter);
// Note: We insert elems into the tree without copying, so there's no special cleanup NicAddressListIter_next(&listIter);
// required for the tree when we're done p++;
PointerRBTree_insert(&tree, nicAddr, NULL);
} }
sort(list, NicAddressList_length(nicList), sizeof(*list), nicaddr_sort_comp, NULL);
// copy elements from tree to clone list (sorted) /* *maybe* the clone already contains elements, don't rely on its size */
p = list;
NicAddressList_init(nicListClone); NicAddressList_init(nicListClone);
while (NicAddressList_length(nicList) != NicAddressList_length(nicListClone)) {
treeIter = PointerRBTree_begin(&tree); NicAddress* clone = os_kmalloc(sizeof(*clone));
memcpy(clone, p->addr, sizeof(*clone));
for( ; !PointerRBTreeIter_end(&treeIter); PointerRBTreeIter_next(&treeIter) ) NicAddressList_append(nicListClone, clone);
{ p++;
NicAddress* nicAddr = (NicAddress*)PointerRBTreeIter_key(&treeIter);
// clone
NicAddress* nicAddrClone = (NicAddress*)os_kmalloc(sizeof(NicAddress) );
memcpy(nicAddrClone, nicAddr, sizeof(NicAddress) );
// append to the clone list
NicAddressList_append(nicListClone, nicAddrClone);
} }
PointerRBTree_uninit(&tree); kfree(list);
} }
void ListTk_kfreeNicAddressListElems(NicAddressList* nicList) void ListTk_kfreeNicAddressListElems(NicAddressList* nicList)
......
...@@ -10,7 +10,8 @@ ...@@ -10,7 +10,8 @@
extern void ListTk_cloneNicAddressList(NicAddressList* nicList, NicAddressList* nicListClone); extern void ListTk_cloneNicAddressList(NicAddressList* nicList, NicAddressList* nicListClone);
extern void ListTk_cloneSortNicAddressList(NicAddressList* nicList, NicAddressList* nicListClone); extern void ListTk_cloneSortNicAddressList(NicAddressList* nicList, NicAddressList* nicListClone,
StrCpyList* preferences);
extern void ListTk_copyUInt16ListToVec(UInt16List* srcList, UInt16Vec* destVec); extern void ListTk_copyUInt16ListToVec(UInt16List* srcList, UInt16Vec* destVec);
extern void ListTk_kfreeNicAddressListElems(NicAddressList* nicList); extern void ListTk_kfreeNicAddressListElems(NicAddressList* nicList);
......
...@@ -56,6 +56,12 @@ int SocketTk_poll(PollState* state, int timeoutMS) ...@@ -56,6 +56,12 @@ int SocketTk_poll(PollState* state, int timeoutMS)
long __timeout = TimeTk_msToJiffiesSchedulable(timeoutMS); long __timeout = TimeTk_msToJiffiesSchedulable(timeoutMS);
/* 4.19 (vanilla, not stable) had a bug in the sock_poll_wait signature. rhel 4.18 backports
* this bug. 4.19.1 fixes it again. */
BUILD_BUG_ON(__builtin_types_compatible_p(
__typeof__(&sock_poll_wait),
void (*)(struct file*, poll_table*)));
poll_initwait(&stdTable); poll_initwait(&stdTable);
if(__timeout) if(__timeout)
......
...@@ -1194,7 +1194,11 @@ outErr: ...@@ -1194,7 +1194,11 @@ outErr:
* global locks enabled. * global locks enabled.
*/ */
int FhgfsOps_atomicOpen(struct inode* dir, struct dentry* dentry, struct file* file, int FhgfsOps_atomicOpen(struct inode* dir, struct dentry* dentry, struct file* file,
unsigned openFlags, umode_t createMode, int* outOpenedFlags) unsigned openFlags, umode_t createMode
#ifndef FMODE_CREATED
, int* outOpenedFlags
#endif
)
{ {
struct super_block* sb = dentry->d_sb; struct super_block* sb = dentry->d_sb;
App* app = FhgfsOps_getApp(sb); App* app = FhgfsOps_getApp(sb);
...@@ -1305,7 +1309,11 @@ int FhgfsOps_atomicOpen(struct inode* dir, struct dentry* dentry, struct file* f ...@@ -1305,7 +1309,11 @@ int FhgfsOps_atomicOpen(struct inode* dir, struct dentry* dentry, struct file* f
{ {
if (lookupOutInfo.createRes == FhgfsOpsErr_SUCCESS) // implies isCreate == true if (lookupOutInfo.createRes == FhgfsOpsErr_SUCCESS) // implies isCreate == true
{ {
#ifdef FMODE_CREATED
file->f_mode |= FMODE_CREATED;
#else
*outOpenedFlags |= FILE_CREATED; *outOpenedFlags |= FILE_CREATED;
#endif
if (lookupOutInfo.lookupRes != FhgfsOpsErr_SUCCESS) if (lookupOutInfo.lookupRes != FhgfsOpsErr_SUCCESS)
{ // only update directory time stamps if the file did not exist yet { // only update directory time stamps if the file did not exist yet
...@@ -1360,7 +1368,11 @@ int FhgfsOps_atomicOpen(struct inode* dir, struct dentry* dentry, struct file* f ...@@ -1360,7 +1368,11 @@ int FhgfsOps_atomicOpen(struct inode* dir, struct dentry* dentry, struct file* f
// stripePattern is assigned to FhgfsInode now, make sure it does not get free'ed // stripePattern is assigned to FhgfsInode now, make sure it does not get free'ed
LookupIntentInfoOut_setStripePattern(&lookupOutInfo, NULL); LookupIntentInfoOut_setStripePattern(&lookupOutInfo, NULL);
retVal = finish_open(file, dentry, generic_file_open, outOpenedFlags); retVal = finish_open(file, dentry, generic_file_open
#ifndef FMODE_CREATED
, outOpenedFlags
#endif
);
if (unlikely(retVal) ) if (unlikely(retVal) )
{ // finish open failed { // finish open failed
int releaseRes; int releaseRes;
......
...@@ -72,7 +72,11 @@ extern int FhgfsOps_mknod(struct inode* dir, struct dentry* dentry, int mode, de ...@@ -72,7 +72,11 @@ extern int FhgfsOps_mknod(struct inode* dir, struct dentry* dentry, int mode, de
#if defined KERNEL_HAS_ATOMIC_OPEN #if defined KERNEL_HAS_ATOMIC_OPEN
int FhgfsOps_atomicOpen(struct inode* dir, struct dentry* dentry, struct file* file, int FhgfsOps_atomicOpen(struct inode* dir, struct dentry* dentry, struct file* file,
unsigned openFlags, umode_t createMode, int* outOpenedFlags); unsigned openFlags, umode_t createMode
#ifndef FMODE_CREATED
, int* outOpenedFlags
#endif
);
extern int FhgfsOps_createIntent(struct inode* dir, struct dentry* dentry, umode_t mode, extern int FhgfsOps_createIntent(struct inode* dir, struct dentry* dentry, umode_t mode,
bool isExclusiveCreate); bool isExclusiveCreate);
#elif defined KERNEL_HAS_UMODE_T #elif defined KERNEL_HAS_UMODE_T
......
...@@ -867,7 +867,7 @@ void FhgfsOpsCommkit_communicate(App* app, RemotingIOInfo* ioInfo, struct list_h ...@@ -867,7 +867,7 @@ void FhgfsOpsCommkit_communicate(App* app, RemotingIOInfo* ioInfo, struct list_h
__commkit_senddata_generic(&context, info); __commkit_senddata_generic(&context, info);
break; break;
} }
/* fall-through if no sendData state is set */ /* fall-through -if no sendData state is set */
case CommKitState_RECVHEADER: case CommKitState_RECVHEADER:
if(context.ops->recvHeader) if(context.ops->recvHeader)
...@@ -875,7 +875,7 @@ void FhgfsOpsCommkit_communicate(App* app, RemotingIOInfo* ioInfo, struct list_h ...@@ -875,7 +875,7 @@ void FhgfsOpsCommkit_communicate(App* app, RemotingIOInfo* ioInfo, struct list_h
__commkit_recvheader_generic(&context, info); __commkit_recvheader_generic(&context, info);
break; break;
} }
/* fall-through if no recvHeader state is set */ /* fall-through -if no recvHeader state is set */
case CommKitState_RECVDATA: case CommKitState_RECVDATA:
if(context.ops->recvData) if(context.ops->recvData)
...@@ -883,7 +883,7 @@ void FhgfsOpsCommkit_communicate(App* app, RemotingIOInfo* ioInfo, struct list_h ...@@ -883,7 +883,7 @@ void FhgfsOpsCommkit_communicate(App* app, RemotingIOInfo* ioInfo, struct list_h
__commkit_recvdata_generic(&context, info); __commkit_recvdata_generic(&context, info);
break; break;
} }
/* fall-through if no recvData state is set */ /* fall-through -if no recvData state is set */
case CommKitState_CLEANUP: case CommKitState_CLEANUP:
__commkit_cleanup_generic(&context, info); __commkit_cleanup_generic(&context, info);
......
...@@ -602,13 +602,13 @@ bool IBVSocket_shutdown(IBVSocket* _this) ...@@ -602,13 +602,13 @@ bool IBVSocket_shutdown(IBVSocket* _this)
{ {
IBVCommContext* commContext = _this->commContext; IBVCommContext* commContext = _this->commContext;
if(_this->errState)
return true; // true, because the conn is down anyways
if(!commContext) if(!commContext)
return true; // this socket has never been connected return true; // this socket has never been connected
if(commContext->incompleteSend.numAvailable) // if object is in errState, then the socket might be in an inconsistent state,
// therefore further commands (except for disconnect) should not be executed
if(!_this->errState && commContext->incompleteSend.numAvailable)
{ // wait for all incomplete sends { // wait for all incomplete sends
int waitRes; int waitRes;
...@@ -1036,6 +1036,19 @@ void __IBVSocket_cleanupCommContext(struct rdma_cm_id* cm_id, IBVCommContext* co ...@@ -1036,6 +1036,19 @@ void __IBVSocket_cleanupCommContext(struct rdma_cm_id* cm_id, IBVCommContext* co
if(!commContext) if(!commContext)
return; return;
if(commContext->qp)
{
// see recommendation here: https://www.rdmamojo.com/2012/12/28/ibv_destroy_qp/
// the qp should be set to error state, so that no more events can be pushed to that queue.
struct ibv_qp_attr qpAttr;
qpAttr.qp_state = IBV_QPS_ERR;
if (ibv_modify_qp(commContext->qp, &qpAttr, IBV_QP_STATE))
{
LOG(SOCKLIB, WARNING, "Failed to modify qp IBV_QP_STATE.");
}
}
// ack remaining delayed acks // ack remaining delayed acks
if(commContext->recvCQ && commContext->numUnackedRecvCompChannelEvents) if(commContext->recvCQ && commContext->numUnackedRecvCompChannelEvents)
ibv_ack_cq_events(commContext->recvCQ, commContext->numUnackedRecvCompChannelEvents); ibv_ack_cq_events(commContext->recvCQ, commContext->numUnackedRecvCompChannelEvents);
......
...@@ -89,37 +89,40 @@ class NetworkInterfaceCard ...@@ -89,37 +89,40 @@ class NetworkInterfaceCard
public: public:
// inliners struct NicAddrComp {
// the comparison implemented here is a model of Compare only if preferences contains
/** // all possible names ever encountered or is unset.
* @return true if lhs (left-hand side) is preferred comared to rhs StringList* preferences = nullptr;
*/
static bool nicAddrPreferenceComp(const NicAddress& lhs, const NicAddress& rhs) explicit NicAddrComp(StringList* preferences = nullptr): preferences(preferences) {}
{
// compares the preference of NICs bool operator()(const NicAddress& lhs, const NicAddress& rhs) const
// returns true if lhs is preferred compared to rhs {
// compares the preference of NICs
// prefer RDMA NICs // returns true if lhs is preferred compared to rhs
if( (lhs.nicType == NICADDRTYPE_RDMA) && (rhs.nicType != NICADDRTYPE_RDMA) ) if (preferences) {
return true; for (const auto& p : *preferences) {
if( (rhs.nicType == NICADDRTYPE_RDMA) && (lhs.nicType != NICADDRTYPE_RDMA) ) if (p == lhs.name)
return false; return true;
if (p == rhs.name)
// prefer SDP NICs return false;
if( (lhs.nicType == NICADDRTYPE_SDP) && (rhs.nicType == NICADDRTYPE_STANDARD) ) }
return true; }
if( (rhs.nicType == NICADDRTYPE_SDP) && (lhs.nicType == NICADDRTYPE_STANDARD) )
return false; // prefer RDMA NICs
if( (lhs.nicType == NICADDRTYPE_RDMA) && (rhs.nicType != NICADDRTYPE_RDMA) )
// prefer higher ipAddr return true;
unsigned lhsHostOrderIP = ntohl(lhs.ipAddr.s_addr); if( (rhs.nicType == NICADDRTYPE_RDMA) && (lhs.nicType != NICADDRTYPE_RDMA) )
unsigned rhsHostOrderIP = ntohl(rhs.ipAddr.s_addr); return false;
// this is the original IP-order version // prefer higher ipAddr
return lhsHostOrderIP > rhsHostOrderIP; unsigned lhsHostOrderIP = ntohl(lhs.ipAddr.s_addr);
return true; unsigned rhsHostOrderIP = ntohl(rhs.ipAddr.s_addr);
}
// this is the original IP-order version
return lhsHostOrderIP > rhsHostOrderIP;
}
};
}; };
......
...@@ -10,13 +10,12 @@ ...@@ -10,13 +10,12 @@
#define NODECONNPOOL_SHUTDOWN_WAITTIMEMS 1500 #define NODECONNPOOL_SHUTDOWN_WAITTIMEMS 1500
/** /**
* @param nicList an internal copy will be created (and sorted) * @param nicList an internal copy will be created
*/ */
LocalNodeConnPool::LocalNodeConnPool(Node& parentNode, NicAddressList& nicList) : LocalNodeConnPool::LocalNodeConnPool(Node& parentNode, NicAddressList& nicList) :
NodeConnPool(parentNode, 0, nicList) NodeConnPool(parentNode, 0, nicList)
{ {
this->nicList = nicList; this->nicList = nicList;
this->nicList.sort(&NetworkInterfaceCard::nicAddrPreferenceComp);
this->establishedConns = 0; this->establishedConns = 0;
this->availableConns = 0; this->availableConns = 0;
......
...@@ -121,7 +121,7 @@ void TargetStateStore::getStatesAndGroups(const MirrorBuddyGroupMapper* buddyGro ...@@ -121,7 +121,7 @@ void TargetStateStore::getStatesAndGroups(const MirrorBuddyGroupMapper* buddyGro
RWLockGuard buddyGroupsLock(buddyGroups->rwlock, SafeRWLock_READ); RWLockGuard buddyGroupsLock(buddyGroups->rwlock, SafeRWLock_READ);