Commit aee03250 authored by Bernd Lietzow's avatar Bernd Lietzow 🐨

updated to release 6.18

parent d4bc30ca
......@@ -77,6 +77,10 @@ sl_unmount_local_mounts()
true # reset error code before next invocation of sl_checkerror
fi
done
# try to remove the client module - this is allowed to fail, because we might have a "normal"
# beegfs mount somewhere in the system.
rmmod beegfs 2>/dev/null || true
}
# sends a SIGTERM to a process, then waits until the process is stopped or appriximately 10 seconds
......
......@@ -360,9 +360,12 @@ void App::handleComponentException(std::exception& e)
const char* logContext = "App::handleComponentException";
LogContext log(logContext);
log.logErr(std::string("This component encountered an unrecoverable error. ") +
std::string("[SysErrnoMessage: ") + System::getErrString() + "] " +
std::string("Exception message: ") + e.what() );
const auto componentName = PThread::getCurrentThreadName();
log.logErr(
"The component [" + componentName + "] encountered an unrecoverable error. " +
std::string("[SysErr: ") + System::getErrString() + "] " +
std::string("Exception message: ") + e.what() );
log.log(Log_WARNING, "Shutting down...");
......
#include "DatagramListener.h"
#include <common/net/message/NetMessageTypes.h>
DatagramListener::DatagramListener(NetFilter* netFilter, NicAddressList& localNicList,
......@@ -17,6 +18,9 @@ void DatagramListener::handleIncomingMsg(struct sockaddr_in* fromAddr, NetMessag
HighResolutionStats stats; // currently ignored
NetMessage::ResponseContext rctx(fromAddr, udpSock, sendBuf, DGRAMMGR_SENDBUF_SIZE, &stats);
NetMsgStrMapping strMapping;
const auto messageType = strMapping.defineToStr(msg->getMsgType());
switch(msg->getMsgType() )
{
// valid messages within this context
......@@ -27,7 +31,9 @@ void DatagramListener::handleIncomingMsg(struct sockaddr_in* fromAddr, NetMessag
case NETMSGTYPE_StatResp:
{
if(!msg->processIncoming(rctx) )
log.log(2, "Problem encountered during handling of incoming message");
{
LOG(WARNING, "Problem encountered during handling of incoming message.", messageType);
}
} break;
default:
......@@ -35,7 +41,7 @@ void DatagramListener::handleIncomingMsg(struct sockaddr_in* fromAddr, NetMessag
log.logErr(
"Received a message that is invalid within the current context "
"from: " + Socket::ipaddrToStr(&fromAddr->sin_addr) + "; "
"type: " + StringTk::intToStr(msg->getMsgType() ) );
"type: " + messageType );
} break;
};
}
......
......@@ -13,7 +13,7 @@ void GetNodeInfoWork::process(char* bufIn, unsigned bufInLen, char* bufOut,
NodeStoreServers* nodes = app->getServerStoreFromType(nodeType);
if(!nodes)
{
log.logErr("Invalid node type: " + Node::nodeTypeToStr(nodeType) );
LOG(ERR, "Invalid node type.", as("Node Type", Node::nodeTypeToStr(nodeType)));
return;
}
......
......@@ -52,7 +52,13 @@ bool HeartbeatMsgEx::processIncoming(ResponseContext& ctx)
default:
{
log.logErr(std::string("Invalid node type: ") + StringTk::intToStr(getNodeType() ) );
LOG(ERR, "Invalid node type.",
as("Node Type", Node::nodeTypeToStr(getNodeType())),
as("Sender", ctx.peerName()),
as("NodeID", getNodeNumID()),
as("Port (UDP)", getPortUDP()),
as("Port (TCP)", getPortTCP())
);
goto ack_resp;
} break;
......
......@@ -131,18 +131,6 @@
<replace file="manifest.mf" value="@VERSION@" token="${VERSION}"/>
</target>
<taskdef name="findbugs" classname="edu.umd.cs.findbugs.anttask.FindBugsTask" />
<target name="findbugs" depends="jar">
<findbugs home="${basedir}/../beegfs_thirdparty/source/findbugs" output="xml"
outputFile="findbugs.xml">
<sourcePath path="${src.dir}" />
<class location="${build.dir}/classes" />
<auxClasspath path="${basedir}/lib/jchart2d-3.2.2.jar" />
<auxClasspath path="${basedir}/lib/jdom.jar" />
<auxClasspath path="${basedir}/lib/nimrodlf-1.2.jar" />
</findbugs>
</target>
<target name="-post-clean">
<condition property="VERSION" value="${BEEGFS_VERSION}" else="DEBUG">
<isset property="BEEGFS_VERSION"/>
......
......@@ -244,6 +244,7 @@ $(call define_if_matches, KERNEL_HAS_I_MMAP_LOCK, -F "i_mmap_lock_read", fs.h)
$(call define_if_matches, KERNEL_HAS_I_MMAP_RWSEM, -F "i_mmap_rwsem", fs.h)
$(call define_if_matches, KERNEL_HAS_I_MMAP_MUTEX, -F "i_mmap_mutex", fs.h)
$(call define_if_matches, KERNEL_HAS_I_MMAP_RBTREE, -P "struct rb_root\s+i_mmap", fs.h)
$(call define_if_matches, KERNEL_HAS_I_MMAP_CACHED_RBTREE, -P "struct rb_root_cached\s+i_mmap", fs.h)
$(call define_if_matches, KERNEL_HAS_I_MMAP_NONLINEAR, -F "i_mmap_nonlinear", fs.h)
$(call define_if_matches, KERNEL_HAS_LONG_IOV_DIO, \
......
......@@ -142,6 +142,11 @@ check_function \
KERNEL_HAS_HAVE_SUBMOUNTS \
linux/dcache.h
check_function \
kernel_read "ssize_t (struct file *, void *, size_t, loff_t *)" \
KERNEL_HAS_KERNEL_READ \
linux/fs.h
# kernel 4.9 and newer have the iov_iter flavor ITER_PIPE which we currently cannot handle in our
# buffered read_iter/write_iter functions. until we can handle it those kernels must have their
# buffered read_iter/write_iter disabled, otherwise the kernel may crash.
......
......@@ -35,6 +35,24 @@ static bool __Config_readLineFromFile(struct file* cfgFile,
static size_t Config_fs_read(struct file *file, char *buf, size_t size, loff_t *pos)
{
size_t readRes;
#if defined(KERNEL_HAS_KERNEL_READ)
readRes = kernel_read(file, buf, size, pos);
#else
mm_segment_t oldfs;
ACQUIRE_PROCESS_CONTEXT(oldfs);
readRes = vfs_read(file, buf, size, pos);
RELEASE_PROCESS_CONTEXT(oldfs);
#endif
return readRes;
}
/**
* @param mountConfig will be copied (not owned by this object)
*/
......@@ -833,7 +851,6 @@ const char* __Config_createDefaultCfgFilename(void)
bool __Config_readLineFromFile(struct file* cfgFile,
char* buf, size_t bufLen, bool* outEndOfFile)
{
mm_segment_t oldfs;
size_t numRead;
bool endOfLine;
bool erroroccurred;
......@@ -842,13 +859,12 @@ bool __Config_readLineFromFile(struct file* cfgFile,
endOfLine = false;
erroroccurred = false;
ACQUIRE_PROCESS_CONTEXT(oldfs);
for(numRead = 0; numRead < (bufLen-1); numRead++)
{
char charBuf;
ssize_t readRes = vfs_read(cfgFile, &charBuf, 1, &cfgFile->f_pos);
ssize_t readRes = Config_fs_read(cfgFile, &charBuf, 1, &cfgFile->f_pos);
if( (readRes > 0) && (charBuf == '\n') )
{ // end of line
......@@ -884,7 +900,7 @@ bool __Config_readLineFromFile(struct file* cfgFile,
{
char charBuf;
ssize_t readRes = vfs_read(cfgFile, &charBuf, 1, &cfgFile->f_pos);
ssize_t readRes = Config_fs_read(cfgFile, &charBuf, 1, &cfgFile->f_pos);
if( (readRes > 0) && (charBuf == '\n') )
endOfLine = true;
if(readRes == 0)
......@@ -895,7 +911,6 @@ bool __Config_readLineFromFile(struct file* cfgFile,
}
RELEASE_PROCESS_CONTEXT(oldfs);
return !erroroccurred;
......@@ -1112,16 +1127,12 @@ bool __Config_initConnAuthHash(Config* this, char* connAuthFile, uint64_t* outCo
return false;
}
readRes = Config_fs_read(fileHandle, buf, CONFIG_AUTHFILE_READSIZE, &fileHandle->f_pos);
ACQUIRE_PROCESS_CONTEXT(oldfs);
readRes = vfs_read(fileHandle, buf, CONFIG_AUTHFILE_READSIZE, &fileHandle->f_pos);
filp_close(fileHandle, NULL);
RELEASE_PROCESS_CONTEXT(oldfs);
if(readRes < 0)
{
printk_fhgfs(KERN_WARNING, "Unable to read auth file: '%s'\n", connAuthFile);
......
......@@ -1222,9 +1222,15 @@ static ssize_t FhgfsOps_buffered_read_iter(struct kiocb *iocb, struct iov_iter *
copyRes = copy_page_to_iter(buffer, 0, readRes, to);
#endif
if (copyRes < readRes)
{
iocb->ki_pos -= (readRes - copyRes);
readRes = copyRes;
}
totalReadRes += readRes;
if (copyRes == 0)
break;
}
}
kunmap(buffer);
......
......@@ -141,7 +141,7 @@ int __FhgfsOps_constructFsInfo(struct super_block* sb, void* rawMountOptions)
#if defined(KERNEL_HAS_SB_BDI)
#if defined(KERNEL_HAS_SB_BDI) && !defined(KERNEL_HAS_SUPER_SETUP_BDI_NAME)
struct backing_dev_info* bdi;
#endif
......@@ -169,23 +169,24 @@ int __FhgfsOps_constructFsInfo(struct super_block* sb, void* rawMountOptions)
IGNORE_UNUSED_VARIABLE(log);
#if defined(KERNEL_HAS_SB_BDI)
bdi = &sbInfo->bdi;
/* NOTE: The kernel expects a fully initialized bdi structure, so at a minimum it has to be
* allocated by kzalloc() or memset(bdi, 0, sizeof(*bdi))
TODO: Those should be set to real functions, as for example in NFS
bdi->congested_fn = NULL;
bdi->congested_data = NULL;
*/
bdi->ra_pages = BEEGFS_DEFAULT_READAHEAD_PAGES;
#if defined(KERNEL_HAS_SUPER_SETUP_BDI_NAME) && !defined(KERNEL_HAS_BDI_SETUP_AND_REGISTER)
res = super_setup_bdi_name(sb, BEEGFS_MODULE_NAME_STR);
#elif defined(KERNEL_HAS_BDI_CAP_MAP_COPY)
res = bdi_setup_and_register(bdi, BEEGFS_MODULE_NAME_STR, BDI_CAP_MAP_COPY);
#else
res = bdi_setup_and_register(bdi, BEEGFS_MODULE_NAME_STR);
bdi = &sbInfo->bdi;
/* NOTE: The kernel expects a fully initialized bdi structure, so at a minimum it has to be
* allocated by kzalloc() or memset(bdi, 0, sizeof(*bdi)).
* we don't set the congest_* callbacks (like every other filesystem) because those are
* intended for dm and md.
*/
bdi->ra_pages = BEEGFS_DEFAULT_READAHEAD_PAGES;
#if defined(KERNEL_HAS_BDI_CAP_MAP_COPY)
res = bdi_setup_and_register(bdi, BEEGFS_MODULE_NAME_STR, BDI_CAP_MAP_COPY);
#else
res = bdi_setup_and_register(bdi, BEEGFS_MODULE_NAME_STR);
#endif
#endif
if (res)
......
......@@ -48,7 +48,9 @@ static inline void FhgfsOps_setIsRootInited(struct super_block* sb, bool isInite
struct FhgfsSuperBlockInfo
{
App app;
#if !defined(KERNEL_HAS_SUPER_SETUP_BDI_NAME)
struct backing_dev_info bdi;
#endif
bool haveRootEntryInfo; // false until the root EntryInfo is set in root-FhgfsInode
bool isRootInited; /* false until root inode attrs have been fetched/initialized in
......@@ -78,9 +80,13 @@ App* FhgfsOps_getApp(struct super_block* sb)
*/
struct backing_dev_info* FhgfsOps_getBdi(struct super_block* sb)
{
#if defined(KERNEL_HAS_SUPER_SETUP_BDI_NAME)
return sb->s_bdi;
#else
FhgfsSuperBlockInfo* sbInfo = sb->s_fs_info;
return &(sbInfo->bdi);
#endif
}
/**
......
......@@ -356,9 +356,12 @@ static inline void i_mmap_unlock_read(struct address_space* mapping)
static inline bool beegfs_hasMappings(struct inode* inode)
{
#ifdef KERNEL_HAS_I_MMAP_RBTREE
#if defined(KERNEL_HAS_I_MMAP_RBTREE)
if (!RB_EMPTY_ROOT(&inode->i_mapping->i_mmap))
return true;
#elif defined(KERNEL_HAS_I_MMAP_CACHED_RBTREE)
if (!RB_EMPTY_ROOT(&inode->i_mapping->i_mmap.rb_root))
return true;
#else
if (!prio_tree_empty(&inode->i_mapping->i_mmap))
return true;
......
......@@ -41,7 +41,7 @@ FhgfsOpsErr MirrorBuddyGroupCreator::addGroup(uint16_t primaryTargetID, uint16_t
errStr = "Mirror buddy group with ID " + StringTk::uintToStr(forcedGroupID)
+ " already exists.";
else
errStr = "Internal error.";
errStr = "Internal error (Please check metadata server log).";
std::string logMessage("Failed to add mirror buddy group: primaryTargetID " +
StringTk::uintToStr(primaryTargetID) + "; secondaryTargetID " +
......
......@@ -227,29 +227,27 @@ bool GetQuotaInfo::requestQuotaDataAndCollectResponses(const NodeHandle& mgmtNod
}
/*
* calculate the number of massages which are required to download all quota data
* calculate the number of messages which are required to download all quota data
*/
int GetQuotaInfo::getMaxMessageCount()
{
int retVal = 1;
unsigned numIds = 1;
if(this->cfg.cfgUseList || this->cfg.cfgUseAll)
{
retVal = this->cfg.cfgIDList.size() / GETQUOTAINFORESPMSG_MAX_ID_COUNT;
if( (this->cfg.cfgIDList.size() % GETQUOTAINFORESPMSG_MAX_ID_COUNT) != 0)
retVal++;
numIds = cfg.cfgIDList.size();
}
else
if(this->cfg.cfgUseRange)
else if (cfg.cfgUseRange)
{
int value = this->cfg.cfgIDRangeEnd - this->cfg.cfgIDRangeStart;
retVal = value / GETQUOTAINFORESPMSG_MAX_ID_COUNT;
if( (value % GETQUOTAINFORESPMSG_MAX_ID_COUNT) != 0)
retVal++;
numIds = this->cfg.cfgIDRangeEnd - this->cfg.cfgIDRangeStart + 1; //inclusive range
}
int retVal = numIds / GETQUOTAINFORESPMSG_MAX_ID_COUNT;
if (numIds % GETQUOTAINFORESPMSG_MAX_ID_COUNT != 0)
++retVal;
return retVal;
}
......@@ -124,6 +124,17 @@ void App::runNormal()
return;
}
// check if running as root
if(geteuid())
{
std::cerr << std::endl
<< "Running beegfs-fsck requires root privileges."
<< std::endl << std::endl << std::endl;
ModeHelp().execute();
appResult = APPCODE_INITIALIZATION_ERROR;
return;
}
initDataObjects(argc, argv);
// wait for mgmtd
......@@ -346,7 +357,8 @@ void App::stopComponents()
// lead to a deadlock (when calling from signal handler)
workersStop();
this->internodeSyncer->selfTerminate();
if(this->internodeSyncer)
this->internodeSyncer->selfTerminate();
if ( dgramListener )
{
......@@ -527,9 +539,12 @@ void App::handleComponentException(std::exception& e)
const char* logContext = "App (component exception handler)";
LogContext log(logContext);
log.logErr(std::string("This component encountered an unrecoverable error. ") +
std::string("[SysErr: ") + System::getErrString() + "] " +
std::string("Exception message: ") + e.what() );
const auto componentName = PThread::getCurrentThreadName();
log.logErr(
"The component [" + componentName + "] encountered an unrecoverable error. " +
std::string("[SysErr: ") + System::getErrString() + "] " +
std::string("Exception message: ") + e.what() );
log.log(2, "Shutting down...");
......
......@@ -288,8 +288,12 @@ void Config::initImplicitVals() throw (InvalidConfigException)
if (!tuneDentryCacheSize)
tuneDentryCacheSize = tuneDbFragmentSize / 384;
// connAuthHash
AbstractConfig::initConnAuthHash(connAuthFile, &connAuthHash);
// read in connAuthFile only if we are running as root.
// if not root, the program will abort anyway
if(!geteuid())
{
AbstractConfig::initConnAuthHash(connAuthFile, &connAuthHash);
}
}
std::string Config::createDefaultCfgFilename()
......
#include "DatagramListener.h"
#include <common/net/message/NetMessageLogHelper.h>
DatagramListener::DatagramListener(NetFilter* netFilter, NicAddressList& localNicList,
AcknowledgmentStore* ackStore, unsigned short udpPort)
throw(ComponentInitException) :
......@@ -16,6 +18,9 @@ void DatagramListener::handleIncomingMsg(struct sockaddr_in* fromAddr, NetMessag
HighResolutionStats stats; // currently ignored
NetMessage::ResponseContext rctx(fromAddr, udpSock, sendBuf, DGRAMMGR_SENDBUF_SIZE, &stats);
NetMsgStrMapping strMapping;
const auto messageType = strMapping.defineToStr(msg->getMsgType());
switch(msg->getMsgType() )
{
// valid messages within this context
......@@ -23,7 +28,9 @@ void DatagramListener::handleIncomingMsg(struct sockaddr_in* fromAddr, NetMessag
case NETMSGTYPE_FsckModificationEvent:
{
if(!msg->processIncoming(rctx) )
log.log(2, "Problem encountered during handling of incoming message");
{
LOG(WARNING, "Problem encountered during handling of incoming message.", messageType);
}
} break;
default:
......@@ -31,7 +38,7 @@ void DatagramListener::handleIncomingMsg(struct sockaddr_in* fromAddr, NetMessag
log.logErr(
"Received a message that is invalid within the current context "
"from: " + Socket::ipaddrToStr(&fromAddr->sin_addr) + "; "
"type: " + StringTk::intToStr(msg->getMsgType() ) );
"type: " + messageType );
} break;
};
}
......
......@@ -53,6 +53,11 @@ void RetrieveInodesWork::process(char* bufIn, unsigned bufInLen, char* bufOut,
void RetrieveInodesWork::doWork(bool isBuddyMirrored)
{
const NumNodeID& metaRootID = Program::getApp()->getMetaNodes()->getRootNodeNumID();
const NumNodeID& nodeID = node.getNumID();
const NumNodeID nodeBuddyGroupID = NumNodeID(Program::getApp()->getMetaMirrorBuddyGroupMapper()
->getBuddyGroupID(node.getNumID().val()));
for ( unsigned firstLevelhashDirNum = hashDirStart; firstLevelhashDirNum <= hashDirEnd;
firstLevelhashDirNum++ )
{
......@@ -135,22 +140,32 @@ void RetrieveInodesWork::doWork(bool isBuddyMirrored)
// check inode entry IDs
for (auto it = dirInodes.begin(); it != dirInodes.end(); )
{
if (db::EntryID::tryFromStr(it->getID()).first
&& db::EntryID::tryFromStr(it->getParentDirID()).first)
auto entryIDPair = db::EntryID::tryFromStr(it->getID());
if (!entryIDPair.first ||
!db::EntryID::tryFromStr(it->getParentDirID()).first)
{
LOG(ERR, "Found inode with invalid entry IDs.",
as("node", it->getSaveNodeID()),
as("isBuddyMirrored", it->getIsBuddyMirrored()),
as("entryID", it->getID()),
as("parentEntryID", it->getParentDirID()));
++it;
errors->increase();
dirInodes.erase(std::prev(it));
continue;
}
LOG(ERR, "Found inode with invalid entry IDs.",
as("node", it->getSaveNodeID()),
as("isBuddyMirrored", it->getIsBuddyMirrored()),
as("entryID", it->getID()),
as("parentEntryID", it->getParentDirID()));
// remove root inodes from non root metas
if (entryIDPair.second.isRootDir() &&
((it->getIsBuddyMirrored() && nodeBuddyGroupID != metaRootID)
|| (!it->getIsBuddyMirrored() && nodeID != metaRootID)))
{
++it;
dirInodes.erase(std::prev(it));
continue;
}
++it;
errors->increase();
dirInodes.erase(std::prev(it));
}
fileInodeCount = fileInodes.size();
......
......@@ -56,6 +56,11 @@ struct EntryID {
return *this == disposal() || *this == mdisposal();
}
bool isRootDir() const
{
return *this == root();
}
std::string str() const
{
if(isSpecial() )
......
......@@ -265,6 +265,11 @@ class FsckDBDentryTable
else
return getPathOf(item.second);
}
void commitChanges()
{
table.commitChanges();
}
};
class FsckDBFileInodesTable
......@@ -362,6 +367,12 @@ class FsckDBFileInodesTable
{
insert(fileInodes, &handle);
}
void commitChanges()
{
inodes.commitChanges();
targets.commitChanges();
}
};
class FsckDBDirInodesTable
......@@ -405,6 +416,11 @@ class FsckDBDirInodesTable
{
insert(fileInodes, &handle);
}
void commitChanges()
{
table.commitChanges();
}
};
class FsckDBChunksTable
......@@ -447,6 +463,11 @@ class FsckDBChunksTable
{
insert(chunks, &handle);
}
void commitChanges()
{
table.commitChanges();
}
};
class FsckDBContDirsTable
......@@ -487,6 +508,11 @@ class FsckDBContDirsTable
{
insert(contDirs, &handle);
}
void commitChanges()
{
table.commitChanges();
}
};
class FsckDBFsIDsTable
......@@ -528,6 +554,11 @@ class FsckDBFsIDsTable
{
insert(fsIDs, &handle);
}
void commitChanges()
{
table.commitChanges();
}
};
template<typename Data>
......
......@@ -163,15 +163,6 @@ int ModeCheckFS::execute()
Config *cfg = app->getConfig();
std::string databasePath = cfg->getDatabasePath();
// check root privileges
if ( geteuid() && getegid() )
{ // no root privileges
FsckTkEx::printVersionHeader(true, true);