XrdClHttp::Filesystem Class Reference

#include <XrdClHttpFilesystem.hh>

Inheritance diagram for XrdClHttp::Filesystem:

Collaboration diagram for XrdClHttp::Filesystem:

Public Member Functions
	Filesystem (const std::string &, std::shared_ptr< HandlerQueue > queue, XrdCl::Log *log)
virtual	~Filesystem () noexcept
XrdCl::XRootDStatus	DirList (const std::string &path, XrdCl::DirListFlags::Flags flags, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual bool	GetProperty (const std::string &name, std::string &value) const override
virtual XrdCl::XRootDStatus	Locate (const std::string &path, XrdCl::OpenFlags::Flags flags, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual XrdCl::XRootDStatus	MkDir (const std::string &path, XrdCl::MkDirFlags::Flags flags, XrdCl::Access::Mode mode, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual XrdCl::XRootDStatus	Prepare (const std::vector< std::string > &fileList, XrdCl::PrepareFlags::Flags flags, uint8_t priority, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual XrdCl::XRootDStatus	Query (XrdCl::QueryCode::Code queryCode, const XrdCl::Buffer &arg, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual XrdCl::XRootDStatus	Rm (const std::string &path, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual XrdCl::XRootDStatus	RmDir (const std::string &path, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual bool	SetProperty (const std::string &name, const std::string &value) override
virtual XrdCl::XRootDStatus	Stat (const std::string &path, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual XrdCl::XRootDStatus	ThirdPartyCopy (const std::string &source, const std::string &dest, const XrdCl::PropertyList *properties, XrdCl::ProgressHandler *progress_handler, time_t timeout=0) override
Public Member Functions inherited from XrdCl::FileSystemPlugIn
virtual	~FileSystemPlugIn ()
	Destructor.
virtual XRootDStatus	ChMod (const std::string &path, Access::Mode mode, ResponseHandler *handler, time_t timeout)
virtual XRootDStatus	DeepLocate (const std::string &path, OpenFlags::Flags flags, ResponseHandler *handler, time_t timeout)
virtual XRootDStatus	DelXAttr (const std::string &path, const std::vector< std::string > &attrs, ResponseHandler *handler, time_t timeout)
virtual XRootDStatus	GetXAttr (const std::string &path, const std::vector< std::string > &attrs, ResponseHandler *handler, time_t timeout)
virtual XRootDStatus	ListXAttr (const std::string &path, ResponseHandler *handler, time_t timeout)
virtual XRootDStatus	Mv (const std::string &source, const std::string &dest, ResponseHandler *handler, time_t timeout)
virtual XRootDStatus	Ping (ResponseHandler *handler, time_t timeout)
virtual XRootDStatus	Protocol (ResponseHandler *handler, time_t timeout=0)
virtual XRootDStatus	SendInfo (const std::string &info, ResponseHandler *handler, time_t timeout)
virtual XRootDStatus	SetXAttr (const std::string &path, const std::vector< xattr_t > &attrs, ResponseHandler *handler, time_t timeout)
virtual XRootDStatus	StatVFS (const std::string &path, ResponseHandler *handler, time_t timeout)
virtual XRootDStatus	Truncate (const std::string &path, uint64_t size, ResponseHandler *handler, time_t timeout)
Public Member Functions inherited from XrdCl::ThirdPartyCopyPlugIn
virtual	~ThirdPartyCopyPlugIn ()

Detailed Description

Definition at line 52 of file XrdClHttpFilesystem.hh.

Constructor & Destructor Documentation

Filesystem()

Filesystem::Filesystem	(	const std::string &	url,
		std::shared_ptr< HandlerQueue >	queue,
		XrdCl::Log *	log )

Definition at line 121 of file XrdClHttpFilesystem.cc.

    : m_queue(queue),
      m_logger(log),
      m_url(url)
{
    m_logger->Debug(kLogXrdClHttp, "Constructing filesystem object with base URL %s", url.c_str());
    // When constructed from the root protocol handler, we've observed it include the
    // path here (the code paths appear to be slightly different from http://).  Strip
    // it out so it's not included twice later.
    m_url.SetPath("/");
    XrdCl::URL::ParamsMap map;
    m_url.SetParams(map);
}

References XrdClHttp::kLogXrdClHttp.

~Filesystem()

Filesystem::~Filesystem ( )

virtualnoexcept

Definition at line 135 of file XrdClHttpFilesystem.cc.

{}

Member Function Documentation

DirList()

XrdCl::XRootDStatus Filesystem::DirList ( const std::string & path, XrdCl::DirListFlags::Flags flags, XrdCl::ResponseHandler * handler, time_t timeout )

overridevirtual

See also

XrdCl::FileSystem::DirlList

Reimplemented from XrdCl::FileSystemPlugIn.

Definition at line 156 of file XrdClHttpFilesystem.cc.

{
    auto ts = XrdClHttp::Factory::GetHeaderTimeoutWithDefault(timeout);
 
    auto full_url = GetCurrentURL(path);
 
    m_logger->Debug(kLogXrdClHttp, "Filesystem::DirList path %s", path.c_str());
    auto listdirOp = std::make_unique<XrdClHttp::CurlListdirOp>(
        handler, full_url,
        m_url.GetHostName() + ":" + std::to_string(m_url.GetPort()),
        SendResponseInfo(), ts, m_logger,
        GetConnCallout(), m_header_callout.load(std::memory_order_acquire));
    return QueueOperation(std::move(listdirOp), "directory list operation");
}

References XrdClHttp::Factory::GetHeaderTimeoutWithDefault(), and XrdClHttp::kLogXrdClHttp.

Here is the call graph for this function:

GetProperty()

bool Filesystem::GetProperty ( const std::string & name, std::string & value ) const

overridevirtual

See also

XrdCl::FileSystem::GetProperty

Reimplemented from XrdCl::FileSystemPlugIn.

Definition at line 193 of file XrdClHttpFilesystem.cc.

{
    std::shared_lock lock(m_properties_mutex);
 
    const auto p = m_properties.find(name);
    if (p == std::end(m_properties)) {
        return false;
    }
 
    value = p->second;
    return true;
}

Locate()

XrdCl::XRootDStatus Filesystem::Locate ( const std::string & path, XrdCl::OpenFlags::Flags flags, XrdCl::ResponseHandler * handler, time_t timeout )

overridevirtual

See also

XrdCl::FileSystem::Locate

Reimplemented from XrdCl::FileSystemPlugIn.

Definition at line 213 of file XrdClHttpFilesystem.cc.

{
    if (!handler) return XrdCl::XRootDStatus();
 
    auto locateInfo = std::make_unique<XrdCl::LocationInfo>();
    locateInfo->Add(XrdCl::LocationInfo::Location(m_url.GetHostName() + ":" + std::to_string(m_url.GetPort()), XrdCl::LocationInfo::ServerOnline, XrdCl::LocationInfo::Read));
 
    auto obj = std::make_unique<XrdCl::AnyObject>();
    obj->Set(locateInfo.release());
    handler->HandleResponse(new XrdCl::XRootDStatus(), obj.release());
 
    return XrdCl::XRootDStatus();
}

References XrdCl::ResponseHandler::HandleResponse(), XrdCl::LocationInfo::Read, and XrdCl::LocationInfo::ServerOnline.

Here is the call graph for this function:

MkDir()

XrdCl::XRootDStatus Filesystem::MkDir ( const std::string & path, XrdCl::MkDirFlags::Flags flags, XrdCl::Access::Mode mode, XrdCl::ResponseHandler * handler, time_t timeout )

overridevirtual

See also

XrdCl::FileSystem::MkDir

Reimplemented from XrdCl::FileSystemPlugIn.

Definition at line 230 of file XrdClHttpFilesystem.cc.

{
    auto ts = XrdClHttp::Factory::GetHeaderTimeoutWithDefault(timeout);
 
    auto full_url = GetCurrentURL(path);
    m_logger->Debug(kLogXrdClHttp, "Filesystem::MkDir path %s", full_url.c_str());
 
    auto mkdirOp = std::make_unique<CurlMkcolOp>(
        handler, full_url, ts, m_logger, SendResponseInfo(), GetConnCallout(),
        m_header_callout.load(std::memory_order_acquire));
    return QueueOperation(std::move(mkdirOp), "filesystem mkdir operation");
}

References XrdClHttp::Factory::GetHeaderTimeoutWithDefault(), and XrdClHttp::kLogXrdClHttp.

Here is the call graph for this function:

Prepare()

XrdCl::XRootDStatus Filesystem::Prepare ( const std::vector< std::string > & fileList, XrdCl::PrepareFlags::Flags flags, uint8_t priority, XrdCl::ResponseHandler * handler, time_t timeout )

overridevirtual

See also

XrdCl::FileSystem::Prepare

Reimplemented from XrdCl::FileSystemPlugIn.

Definition at line 247 of file XrdClHttpFilesystem.cc.

{
    (void)priority;
 
    if(fileList.empty())
    {
        return XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidArgs,
            EINVAL, "missing prepare file list");
    }
 
    const auto ts = XrdClHttp::Factory::GetHeaderTimeoutWithDefault(timeout);
    auto tapeOp = std::make_unique<CurlTapePrepareOp>(
        handler, m_url.GetURL(), fileList, flags, ts, m_logger,
        GetConnCallout(),
        m_header_callout.load(std::memory_order_acquire));
    return QueueOperation(std::move(tapeOp), "Tape prepare operation");
}

References XrdCl::errInvalidArgs, XrdClHttp::Factory::GetHeaderTimeoutWithDefault(), and XrdCl::stError.

Here is the call graph for this function:

Query()

XrdCl::XRootDStatus Filesystem::Query ( XrdCl::QueryCode::Code queryCode, const XrdCl::Buffer & arg, XrdCl::ResponseHandler * handler, time_t timeout )

overridevirtual

See also

XrdCl::FileSystem::Query

Reimplemented from XrdCl::FileSystemPlugIn.

Definition at line 270 of file XrdClHttpFilesystem.cc.

{
    const auto ts = XrdClHttp::Factory::GetHeaderTimeoutWithDefault(timeout);
    std::unique_ptr<CurlOperation> operation;
    const char *description = nullptr;
 
    switch(queryCode)
    {
        case XrdCl::QueryCode::Prepare:
        case XrdCl::QueryCode::Opaque:
            operation = std::make_unique<CurlTapeQueryOp>(
                handler, m_url.GetURL(), queryCode, arg, ts, m_logger,
                GetConnCallout(),
                m_header_callout.load(std::memory_order_acquire));
            description = "Tape query operation";
            break;
        case XrdCl::QueryCode::Checksum:
        {
            const auto url = GetCurrentURL(arg.ToString());
            m_logger->Debug(kLogXrdClHttp,
                "XrdClHttp::Filesystem::Query checksum path %s", url.c_str());
 
            XrdClHttp::ChecksumType preferred =
                XrdClHttp::ChecksumType::kAll;
            XrdCl::URL urlObj;
            urlObj.FromString(url);
            const auto iter = urlObj.GetParams().find("cks.type");
            if(iter != urlObj.GetParams().end())
            {
                preferred = XrdClHttp::GetTypeFromString(iter->second);
                if(preferred == XrdClHttp::ChecksumType::kUnknown)
                {
                    m_logger->Error(kLogXrdClHttp,
                        "Unknown checksum type %s", iter->second.c_str());
                    return XrdCl::XRootDStatus(XrdCl::stError,
                        XrdCl::errInvalidArgs, EINVAL,
                        "unknown checksum type '" + iter->second + "'");
                }
            }
            operation = std::make_unique<CurlChecksumOp>(
                handler, url, preferred, ts, m_logger, SendResponseInfo(),
                GetConnCallout(),
                m_header_callout.load(std::memory_order_acquire));
            description = "checksum operation";
            break;
        }
        case XrdCl::QueryCode::XAttr:
        {
            const std::string path = arg.ToString();
            m_logger->Debug(kLogXrdClHttp,
                "XrdClHttp::Filesystem::Query xattr full_url %s, path %s",
                m_url.GetURL().c_str(), path.c_str());
            operation = std::make_unique<CurlQueryOp>(
                handler, path, ts, m_logger, SendResponseInfo(),
                GetConnCallout(), queryCode,
                m_header_callout.load(std::memory_order_acquire));
            description = "xattr query operation";
            break;
        }
        default:
            return XrdCl::XRootDStatus(
                XrdCl::stError, XrdCl::errNotImplemented);
    }
 
    return QueueOperation(std::move(operation), description);
}

References XrdCl::QueryCode::Checksum, XrdCl::errInvalidArgs, XrdCl::errNotImplemented, XrdCl::URL::FromString(), XrdClHttp::Factory::GetHeaderTimeoutWithDefault(), XrdCl::URL::GetParams(), XrdClHttp::GetTypeFromString(), XrdClHttp::kAll, XrdClHttp::kLogXrdClHttp, XrdClHttp::kUnknown, XrdCl::QueryCode::Opaque, XrdCl::QueryCode::Prepare, XrdCl::stError, XrdCl::Buffer::ToString(), and XrdCl::QueryCode::XAttr.

Here is the call graph for this function:

Rm()

XrdCl::XRootDStatus Filesystem::Rm ( const std::string & path, XrdCl::ResponseHandler * handler, time_t timeout )

overridevirtual

See also

XrdCl::FileSystem::Rm

Reimplemented from XrdCl::FileSystemPlugIn.

Definition at line 341 of file XrdClHttpFilesystem.cc.

{
    auto ts = XrdClHttp::Factory::GetHeaderTimeoutWithDefault(timeout);
 
    auto full_url = GetCurrentURL(path);
    m_logger->Debug(kLogXrdClHttp, "Filesystem::Rm path %s", full_url.c_str());
 
    auto deleteOp = std::make_unique<CurlDeleteOp>(
        handler, full_url, ts, m_logger, SendResponseInfo(),
        GetConnCallout(), m_header_callout.load(std::memory_order_acquire));
    return QueueOperation(std::move(deleteOp), "filesystem delete operation");
}

References XrdClHttp::Factory::GetHeaderTimeoutWithDefault(), and XrdClHttp::kLogXrdClHttp.

Referenced by RmDir().

Here is the call graph for this function:

Here is the caller graph for this function:

RmDir()

XrdCl::XRootDStatus Filesystem::RmDir ( const std::string & path, XrdCl::ResponseHandler * handler, time_t timeout )

overridevirtual

See also

XrdCl::FileSystem::RmDir

Reimplemented from XrdCl::FileSystemPlugIn.

Definition at line 357 of file XrdClHttpFilesystem.cc.

{
    return Rm(path, handler, timeout);
}

References Rm().

Here is the call graph for this function:

SetProperty()

bool Filesystem::SetProperty ( const std::string & name, const std::string & value )

overridevirtual

See also

XrdCl::FileSystem::SetProperty

Reimplemented from XrdCl::FileSystemPlugIn.

Definition at line 365 of file XrdClHttpFilesystem.cc.

{
    if (name == "XrdClHttpHeaderCallout") {
        long long pointer;
        try {
            pointer = std::stoll(value, nullptr, 16);
        } catch (...) {
            pointer = 0;
        }
        if (!pointer) {
            pointer = 0;
        }
        m_header_callout.store(reinterpret_cast<XrdClHttp::HeaderCallout*>(pointer), std::memory_order_release);
    }
 
    std::unique_lock lock(m_properties_mutex);
    m_properties[name] = value;
    return true;
}

Stat()

XrdCl::XRootDStatus Filesystem::Stat ( const std::string & path, XrdCl::ResponseHandler * handler, time_t timeout )

overridevirtual

See also

XrdCl::FileSystem::Stat

Reimplemented from XrdCl::FileSystemPlugIn.

Definition at line 387 of file XrdClHttpFilesystem.cc.

{
    auto ts = XrdClHttp::Factory::GetHeaderTimeoutWithDefault(timeout);
 
    auto full_url = GetCurrentURL(path);
    m_logger->Debug(kLogXrdClHttp, "Filesystem::Stat path %s", full_url.c_str());
 
    auto statOp = std::make_unique<CurlStatOp>(
        handler, full_url, ts, m_logger, SendResponseInfo(),
        GetConnCallout(), m_header_callout.load(std::memory_order_acquire));
    return QueueOperation(std::move(statOp), "filesystem stat operation");
}

References XrdClHttp::Factory::GetHeaderTimeoutWithDefault(), and XrdClHttp::kLogXrdClHttp.

Here is the call graph for this function:

ThirdPartyCopy()

XrdCl::XRootDStatus Filesystem::ThirdPartyCopy ( const std::string & source, const std::string & dest, const XrdCl::PropertyList * properties, XrdCl::ProgressHandler * progress_handler, time_t timeout = 0 )

overridevirtual

Third party copy

The copy runs to completion before this method returns.

Parameters

source	the file to be copied from
dest	the location the file is copied to
properties	properties of the copy, none if null
progress_handler	handler notified while the copy runs, none if null
timeout	timeout value, if 0 the environment default is used

Returns

status of the operation

Implements XrdCl::ThirdPartyCopyPlugIn.

Definition at line 462 of file XrdClHttpFilesystem.cc.

{
    XrdCl::Log *const log = XrdCl::DefaultEnv::GetLog();
    log->Debug(kLogXrdClHttp, "XrdClHttp::ThirdPartyCopy src %s dst %s", source.c_str(), dest.c_str());
 
    if (!is_http_url(source) || !is_http_url(dest))
    {
        log->Error(kLogXrdClHttp, "Third party copy can only be done between http(s) protocols");
        return XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidArgs, 0,
                                   "Third party copy can only be done between http(s) protocols");
    }
 
    TpcMode mode = TpcMode::Pull;
    int number_of_streams = 1;
    time_t init_timeout = 0;
    time_t tpc_timeout = 0;
    std::string token_file;
    bool force_overwrite = false;
 
    if (properties)
    {
        mode = properties->Get<std::string>("thirdPartyMode") != "push" ? TpcMode::Pull : TpcMode::Push;
        properties->Get("initTimeout", init_timeout);
        properties->Get("tpcTimeout", tpc_timeout);
        token_file = properties->Get<std::string>("thirdPartyTokenFile");
        properties->Get("force", force_overwrite);
    }
 
    // The tpcTimeout property takes precedence over the timeout argument.
    if (tpc_timeout == 0)
        tpc_timeout = timeout;
 
    if (auto env = XrdCl::DefaultEnv::GetEnv(); env)
        env->GetInt("SubStreamsPerChannel", number_of_streams);
 
    CurlCopyOp::Headers headers;
    CurlCopyOp::Headers src_hdrs;
    CurlCopyOp::Headers dst_hdrs;
 
    if (!token_file.empty() && !ParseTokenFile(token_file, src_hdrs, dst_hdrs))
        return XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errAuthFailed, 0,
                                   "Failed to parse the token file '" + token_file + "'");
 
    const auto is_unsecure = [] (const CurlCopyOp::Headers &hdrs, const std::string &url)
    {
        return XrdCl::URL(url).GetProtocol() != "https"s &&
               std::any_of(hdrs.begin(), hdrs.end(),
                           [] (const auto &header) { return header.first == "Authorization"s; });
    };
 
    if (is_unsecure(src_hdrs, source) || is_unsecure(dst_hdrs, dest))
    {
        log->Error(kLogXrdClHttp, "Refusing to send an Authorization header over an unencrypted http URL");
        return XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidArgs, 0,
                                   "Refusing to send an Authorization header over an unencrypted http URL");
    }
 
    // In push mode the source writes the destination with a PUT, which may
    // replace an existing file whatever the Overwrite header says. Thus check
    // the destination first, as the destination server may not honor it.
    if (mode == TpcMode::Push && !force_overwrite &&
        stat_size(*m_queue, dest, dst_hdrs, init_timeout, log, "destination"))
    {
        log->Error(kLogXrdClHttp, "The destination %s exists", dest.c_str());
        return XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errErrorResponse, kXR_ItExists,
                                   "The destination exists; use --force to replace it");
    }
 
    headers.emplace_back("X-Number-Of-Streams"s, std::to_string(number_of_streams));
 
    // The destination headers reach the destination in both modes: directly
    // in pull mode, with the TransferHeader prefix in push mode.
    dst_hdrs.emplace_back("Overwrite"s, force_overwrite ? "T"s : "F"s);
 
    std::size_t size = 0;
 
    if (progress_handler)
    {
        size = stat_size(*m_queue, source, src_hdrs, init_timeout, log, "source").value_or(0);
        progress_handler->HandleProgress(0, size);
    }
 
    const auto rh = std::make_shared<SyncResponseHandler>();
    const auto op = std::make_shared<CopyOp>(rh.get(), source, src_hdrs, dest, dst_hdrs, headers,
                                             mode, timespec{tpc_timeout, 0}, log, nullptr);
    op->SetProgressHandler(progress_handler);
 
    try
    {
        m_queue->Produce(std::shared_ptr<CopyOp>(op.get(), [rh, op](auto){}));
    }
    catch (const std::exception &e) {
        log->Warning(kLogXrdClHttp, "Failed to add copy op to queue - %s", e.what());
        return XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInternal);
    }
 
    if (!rh->wait(std::chrono::seconds(tpc_timeout)))
        return XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errOperationExpired, 0, "Operation expired: Operation timed out"s);
 
    if (op->IsDone() && !op->IsSentSuccessfully())
        return XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errPipelineFailed, 0, op->GetSendingFailureMessage());
 
    if (progress_handler && size > 0)
        progress_handler->HandleProgress(size, size);
 
    return XrdCl::XRootDStatus();
}

References XrdCl::Log::Debug(), XrdCl::errAuthFailed, XrdCl::errErrorResponse, XrdCl::errInternal, XrdCl::errInvalidArgs, XrdCl::errOperationExpired, XrdCl::Log::Error(), XrdCl::errPipelineFailed, XrdCl::PropertyList::Get(), XrdCl::DefaultEnv::GetEnv(), XrdCl::DefaultEnv::GetLog(), XrdCl::URL::GetProtocol(), XrdCl::ProgressHandler::HandleProgress(), is_http_url(), XrdClHttp::kLogXrdClHttp, kXR_ItExists, XrdClHttp::ParseTokenFile(), XrdClHttp::Pull, XrdClHttp::Push, XrdClHttp::stat_size(), XrdCl::stError, and XrdCl::Log::Warning().

Here is the call graph for this function:

---

The documentation for this class was generated from the following files:

• XrdClHttpFilesystem.hh
• XrdClHttpFilesystem.cc