XrdCl::ZipArchive Class Reference

#include <XrdClZipArchive.hh>

Collaboration diagram for XrdCl::ZipArchive:

Public Member Functions
	ZipArchive (bool enablePlugIns=true)
	Constructor.
virtual	~ZipArchive ()
	Destructor.
XRootDStatus	AppendFile (const std::string &fn, uint32_t crc32, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
XRootDStatus	CloseArchive (ResponseHandler *handler, uint16_t timeout=0)
	Create the central directory at the end of ZIP archive and close it.
XRootDStatus	CloseFile ()
XRootDStatus	GetCRC32 (const std::string &fn, uint32_t &cksum)
File &	GetFile ()
	Get the underlying File object.
XRootDStatus	GetOffset (const std::string &fn, uint64_t &offset)
bool	GetProperty (const std::string &name, std::string &value)
	Get property on the underlying File object.
bool	IsOpen ()
bool	IsSecure ()
	Check if the underlying file is using an encrypted connection.
XRootDStatus	List (DirectoryList *&list)
XRootDStatus	OpenArchive (const std::string &url, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout=0)
XRootDStatus	OpenFile (const std::string &fn, OpenFlags::Flags flags=OpenFlags::None, uint64_t size=0, uint32_t crc32=0)
XRootDStatus	PgRead (uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
XRootDStatus	PgReadFrom (const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
XRootDStatus	Read (uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
XRootDStatus	ReadFrom (const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
bool	SetProperty (const std::string &name, const std::string &value)
	Set property on the underlying File object.
XRootDStatus	Stat (const std::string &fn, StatInfo *&info)
XRootDStatus	Stat (StatInfo *&info)
XRootDStatus	UpdateMetadata (uint32_t crc32)
XRootDStatus	Write (uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)

Friends
class	::MicroTest
class	::XrdEcTests
template<typename RSP>
XRootDStatus	ReadFromImpl (ZipArchive &me, const std::string &fn, uint64_t relativeOffset, uint32_t size, void *usrbuff, ResponseHandler *usrHandler, uint16_t timeout)
template<bool>
class	XrdEc::OpenOnlyImpl
class	XrdEc::Reader
class	XrdEc::StrmWriter

Detailed Description

Definition at line 60 of file XrdClZipArchive.hh.

Constructor & Destructor Documentation

ZipArchive()

XrdCl::ZipArchive::ZipArchive

(

bool

enablePlugIns = true

)

Constructor.

Definition at line 208 of file XrdClZipArchive.cc.

                                            : archive( enablePlugIns ),
                                                archsize( 0 ),
                                                cdexists( false ),
                                                updated( false ),
                                                cdoff( 0 ),
                                                orgcdsz( 0 ),
                                                orgcdcnt( 0 ),
                                                openstage( None ),
                                                ckpinit( false )
  {
  }

Referenced by ::XrdEcTests, and ReadFromImpl.

Here is the caller graph for this function:

~ZipArchive()

XrdCl::ZipArchive::~ZipArchive ( )

virtual

Destructor.

Definition at line 223 of file XrdClZipArchive.cc.

  {
  }

Member Function Documentation

AppendFile()

XRootDStatus XrdCl::ZipArchive::AppendFile	(	const std::string &	fn,
		uint32_t	crc32,
		uint32_t	size,
		const void *	buffer,
		ResponseHandler *	handler,
		uint16_t	timeout = 0 )

Create a new file in the ZIP archive and append the data

Parameters

fn	: the name of the new file to be created
crc32	: the crc32 of the file
size	: the size of the file
buffer	: the buffer with the data
handler	: user callback
timeout	: operation timeout

Returns

: the status of the operation

Definition at line 877 of file XrdClZipArchive.cc.

  {
    Log  *log = DefaultEnv::GetLog();
    auto  itr   = cdmap.find( fn );
    // check if the file already exists in the archive
    if( itr != cdmap.end() )
    {
      log->Dump( ZipMsg, "[%p] Open failed: file exists %s, cannot append.",
                         (void*)this, fn.c_str() );
      return XRootDStatus( stError, errInvalidOp );
    }
 
    log->Dump( ZipMsg, "[%p] Appending file: %s.", (void*)this, fn.c_str() );
    //-------------------------------------------------------------------------
    // Create Local File Header record
    //-------------------------------------------------------------------------
    lfh.reset( new LFH( fn, crc32, size, time( 0 ) ) );
    //-------------------------------------------------------------------------
    // And write it all
    //-------------------------------------------------------------------------
    return WriteImpl( size, buffer, handler, timeout );
  }

References XrdCl::Log::Dump(), XrdCl::errInvalidOp, XrdCl::DefaultEnv::GetLog(), XrdCl::stError, and XrdCl::ZipMsg.

Here is the call graph for this function:

CloseArchive()

XRootDStatus XrdCl::ZipArchive::CloseArchive	(	ResponseHandler *	handler,
		uint16_t	timeout = 0 )

Create the central directory at the end of ZIP archive and close it.

Parameters

handler	: user callback
timeout	: operation timeout

Returns

: the status of the operation

Definition at line 604 of file XrdClZipArchive.cc.

  {
    Log *log = DefaultEnv::GetLog();
 
    //-------------------------------------------------------------------------
    // If the file was updated, we need to write the Central Directory before
    // closing the file.
    //-------------------------------------------------------------------------
    if( updated )
    {
      ChunkList chunks;
      std::vector<std::shared_ptr<buffer_t>> wrtbufs;
      for( auto &p : newfiles )
      {
        NewFile &nf = p.second;
        if( !nf.overwrt ) continue;
        uint32_t lfhlen = lfh->lfhSize;
        auto lfhbuf = std::make_shared<buffer_t>();
        lfhbuf->reserve( lfhlen );
        nf.lfh->Serialize( *lfhbuf );
        chunks.emplace_back( nf.offset, lfhbuf->size(), lfhbuf->data() );
        wrtbufs.emplace_back( std::move( lfhbuf ) );
      }
 
      auto wrtbuff = std::make_shared<buffer_t>( GetCD() );
      Pipeline p = XrdCl::Write( archive, cdoff,
                                 wrtbuff->size(),
                                 wrtbuff->data() );
      wrtbufs.emplace_back( std::move( wrtbuff ) );
 
      std::vector<ChunkList> listsvec;
      XrdCl::Utils::SplitChunks( listsvec, chunks, 262144, 1024 );
 
      for(auto itr = listsvec.rbegin(); itr != listsvec.rend(); ++itr)
      {
        p = XrdCl::VectorWrite( archive, *itr ) | p;
      }
      if( ckpinit )
        p       |= XrdCl::Checkpoint( archive, ChkPtCode::COMMIT );
      p         |= Close( archive ) >>
                     [this]( XRootDStatus &st )
                     {
                       if( st.IsOK() ) this->Clear();
                       else openstage = Error;
                     }
                 | XrdCl::Final( [handler,log,wrtbufs,this]( const XRootDStatus &st ) mutable
                     {
                       if( st.IsOK() )
                         log->Dump( ZipMsg, "[%p] Successfully closed ZIP archive "
                                            "(CD written).", (void*)this );
                       else
                         log->Error( ZipMsg, "[%p] Failed to close ZIP archive: %s",
                                             (void*)this, st.ToString().c_str() );
                       wrtbufs.clear();
                       if( handler ) handler->HandleResponse( make_status( st ), nullptr );
                     } );
 
      Async( std::move( p ), timeout );
      return XRootDStatus();
    }
 
    //-------------------------------------------------------------------------
    // Otherwise, just close the ZIP archive
    //-------------------------------------------------------------------------
    Pipeline p = Close( archive ) >>
                   [handler,log,this]( XRootDStatus &st )
                   {
                     if( st.IsOK() )
                     {
                       this->Clear();
                       log->Dump( ZipMsg, "[%p] Successfully closed "
                                          "ZIP archive.", (void*)this );
                     }
                     else
                     {
                       openstage = Error;
                       log->Error( ZipMsg, "[%p] Failed to close ZIP archive:"
                                           " %s", (void*)this, st.ToString().c_str() );
                     }
                     if( handler )
                       handler->HandleResponse( make_status( st ), nullptr );
                   };
    Async( std::move( p ), timeout );
    return XRootDStatus();
  }

References XrdCl::Async(), XrdCl::Checkpoint(), XrdCl::Close(), XrdCl::COMMIT, XrdCl::Log::Dump(), XrdCl::Log::Error(), XrdCl::DefaultEnv::GetLog(), XrdCl::ResponseHandler::HandleResponse(), XrdCl::Status::IsOK(), XrdCl::Utils::SplitChunks(), XrdCl::Status::ToString(), XrdCl::VectorWrite(), XrdCl::Write(), and XrdCl::ZipMsg.

Here is the call graph for this function:

CloseFile()

XRootDStatus XrdCl::ZipArchive::CloseFile ( )

inline

Close an open file within the ZIP archive

Returns

: the status of the operation

Definition at line 339 of file XrdClZipArchive.hh.

      {
        if( openstage != Done || openfn.empty() )
          return XRootDStatus( stError, errInvalidOp,
                               0, "Archive not opened." );
        openfn.clear();
        lfh.reset();
        return XRootDStatus();
      }

References XrdCl::errInvalidOp, and XrdCl::stError.

GetCRC32()

XRootDStatus XrdCl::ZipArchive::GetCRC32 ( const std::string & fn, uint32_t & cksum )

inline

Get crc32 for a given file

Parameters

fn	: file name
cksum	: output parameter

Returns

: the status of the operation

Definition at line 278 of file XrdClZipArchive.hh.

      { // make sure archive has been opened and CD has been parsed
        if( openstage != Done )
          return XRootDStatus( stError, errInvalidOp );
        // make sure the file is part of the archive
        auto cditr = cdmap.find( fn );
        if( cditr == cdmap.end() )
          return XRootDStatus( stError, errNotFound );
        cksum = cdvec[cditr->second]->ZCRC32;
        return XRootDStatus();
      }

References XrdCl::errInvalidOp, XrdCl::errNotFound, and XrdCl::stError.

GetFile()

File & XrdCl::ZipArchive::GetFile ( )

inline

Get the underlying File object.

Definition at line 390 of file XrdClZipArchive.hh.

      {
        return archive;
      }

GetOffset()

XRootDStatus XrdCl::ZipArchive::GetOffset ( const std::string & fn, uint64_t & offset )

inline

Definition at line 290 of file XrdClZipArchive.hh.

                                                                           {
          if( openstage != XrdCl::ZipArchive::Done || !archive.IsOpen() )
                        return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errInvalidOp );
 
                      auto cditr = cdmap.find( fn );
                      if( cditr == cdmap.end() )
                        return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotFound,
                                        XrdCl::errNotFound, "File not found." );
 
                      XrdCl::CDFH *cdfh = cdvec[cditr->second].get();
 
                      // check if the file is compressed, for now we only support uncompressed and inflate/deflate compression
                      if( cdfh->compressionMethod != 0 && cdfh->compressionMethod != Z_DEFLATED )
                        return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotSupported,
                                             0, "The compression algorithm is not supported!" );
 
                      // Now the problem is that at the beginning of our
                      // file there is the Local-file-header, which size
                      // is not known because of the variable size 'extra'
                      // field, so we need to know the offset of the next
                      // record and shift it by the file size.
                      // The next record is either the next LFH (next file)
                      // or the start of the Central-directory.
                      uint64_t cdOffset = zip64eocd ? zip64eocd->cdOffset : eocd->cdOffset;
                      uint64_t nextRecordOffset = ( cditr->second + 1 < cdvec.size() ) ?
                                  XrdCl::CDFH::GetOffset( *cdvec[cditr->second + 1] ) : cdOffset;
                      uint64_t filesize = cdfh->compressedSize;
                      if( filesize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
                        filesize = cdfh->extra->compressedSize;
                      uint16_t descsize = cdfh->HasDataDescriptor() ?
                                  XrdCl::DataDescriptor::GetSize( cdfh->IsZIP64() ) : 0;
                      offset  = nextRecordOffset - filesize - descsize;
                      return XrdCl::XRootDStatus();
      }

References XrdZip::CDFH::compressedSize, XrdZip::CDFH::compressionMethod, XrdCl::errInvalidOp, XrdCl::errNotFound, XrdCl::errNotSupported, XrdZip::CDFH::extra, XrdZip::CDFH::HasDataDescriptor(), XrdZip::CDFH::IsZIP64(), and XrdCl::stError.

Referenced by ReadFromImpl.

Here is the call graph for this function:

Here is the caller graph for this function:

GetProperty()

bool XrdCl::ZipArchive::GetProperty ( const std::string & name, std::string & value )

inline

Get property on the underlying File object.

Definition at line 382 of file XrdClZipArchive.hh.

      {
        return archive.GetProperty( name, value );
      }

References XrdCl::NullRef< Response >::value.

IsOpen()

bool XrdCl::ZipArchive::IsOpen ( )

inline

Returns

: true if ZIP archive has been successfully opened

Definition at line 358 of file XrdClZipArchive.hh.

      {
        return openstage == Done;
      }

IsSecure()

bool XrdCl::ZipArchive::IsSecure ( )

inline

Check if the underlying file is using an encrypted connection.

Definition at line 366 of file XrdClZipArchive.hh.

      {
        return archive.IsSecure();
      }

List()

XRootDStatus XrdCl::ZipArchive::List

(

DirectoryList *&

list

)

List files in the ZIP archive

Returns

: the status of the operation

Definition at line 720 of file XrdClZipArchive.cc.

  {
    if( openstage != Done )
      return XRootDStatus( stError, errInvalidOp,
                           0, "Archive not opened." );
 
    std::string value;
    archive.GetProperty( "LastURL", value );
    URL url( value );
 
    StatInfo *infoptr = 0;
    XRootDStatus st = archive.Stat( false, infoptr );
    std::unique_ptr<StatInfo> info( infoptr );
 
    list = new DirectoryList();
    list->SetParentName( url.GetPath() );
 
    auto itr = cdvec.begin();
    for( ; itr != cdvec.end() ; ++itr )
    {
      CDFH *cdfh = itr->get();
      uint64_t uncompressedSize = cdfh->uncompressedSize;
      if( uncompressedSize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
        uncompressedSize = cdfh->extra->uncompressedSize;
      StatInfo *entry_info = make_stat( *info, uncompressedSize );
      DirectoryList::ListEntry *entry =
          new DirectoryList::ListEntry( url.GetHostId(), cdfh->filename, entry_info );
      list->Add( entry );
    }
 
    return XRootDStatus();
  }

References XrdCl::DirectoryList::Add(), XrdCl::errInvalidOp, XrdZip::CDFH::extra, XrdZip::CDFH::filename, XrdCl::URL::GetHostId(), XrdCl::URL::GetPath(), XrdCl::NullRef< Response >::value, XrdCl::DirectoryList::SetParentName(), XrdCl::stError, and XrdZip::CDFH::uncompressedSize.

Here is the call graph for this function:

OpenArchive()

XRootDStatus XrdCl::ZipArchive::OpenArchive	(	const std::string &	url,
		OpenFlags::Flags	flags,
		ResponseHandler *	handler,
		uint16_t	timeout = 0 )

Open ZIP Archive (and parse the Central Directory)

Parameters

url	: the URL of the ZIP archive
flags	: open flags to be used when openning the file
handler	: user callback
timeout	: operation timeout

Returns

: the status of the operation

Definition at line 266 of file XrdClZipArchive.cc.

  {
    Log *log = DefaultEnv::GetLog();
    Fwd<uint32_t> rdsize; // number of bytes to be read
    Fwd<uint64_t> rdoff;  // offset for the read request
    Fwd<void*>    rdbuff; // buffer for data to be read
    uint32_t      maxrdsz = EOCD::maxCommentLength + EOCD::eocdBaseSize +
                            ZIP64_EOCDL::zip64EocdlSize;
 
    Pipeline open_archive = // open the archive
                            XrdCl::Open( archive, url, flags ) >>
                              [log,rdsize,rdoff,rdbuff,maxrdsz,this]( XRootDStatus &status, StatInfo &info ) mutable
                              {
                                 // check the status is OK
                                 if( !status.IsOK() ) return;
 
                                 archsize = info.GetSize();
                                 // if it is an empty file (possibly a new file) there's nothing more to do
                                 if( archsize == 0 )
                                 {
                                   cdexists = false;
                                   openstage = Done;
                                   log->Dump( ZipMsg, "[%p] Opened a ZIP archive (file empty).", (void*)this );
                                   Pipeline::Stop();
                                 }
                                 // prepare the arguments for the subsequent read
                                 rdsize = ( archsize <= maxrdsz ? archsize : maxrdsz );
                                 rdoff  = archsize - *rdsize;
                                 buffer.reset( new char[*rdsize] );
                                 rdbuff = buffer.get();
                                 openstage = HaveEocdBlk;
                                 log->Dump( ZipMsg, "[%p] Opened a ZIP archive, reading "
                                                    "Central Directory at offset: %llu.", (void*)this, (unsigned long long) *rdoff );
                               }
                            // read the Central Directory (in several stages if necessary)
                          | XrdCl::Read( archive, rdoff, rdsize, rdbuff ) >>
                              [log,rdoff,rdsize,rdbuff,this]( XRootDStatus &status, ChunkInfo &chunk ) mutable
                              {
                                // check the status is OK
                                if( !status.IsOK() ) return;
 
                                const char *buff = reinterpret_cast<char*>( chunk.buffer );
                                while( true )
                                {
                                  switch( openstage )
                                  {
                                    case HaveEocdBlk:
                                    {
                                      // Parse the EOCD record
                                      const char *eocdBlock = EOCD::Find( buff, chunk.length );
                                      if( !eocdBlock )
                                      {
                                        XRootDStatus error( stError, errDataError, 0,
                                                            "End-of-central-directory signature not found." );
                                        Pipeline::Stop( error );
                                      }
                                      try{
                                      eocd.reset( new EOCD( eocdBlock, chunk.length - uint32_t(eocdBlock - buff) ) );
                                      log->Dump( ZipMsg, "[%p] EOCD record parsed: %s", (void*)this,
                                                         eocd->ToString().c_str() );
                                      if(eocd->cdOffset > archsize || eocd->cdOffset + eocd->cdSize > archsize)
                                        throw bad_data();
                                      }
                                      catch(const bad_data &ex){
                                        XRootDStatus error( stError, errDataError, 0,
                                                            "End-of-central-directory signature corrupted." );
                                        Pipeline::Stop( error );
                                      }
                                      // Do we have the whole archive?
                                      if( chunk.length == archsize )
                                      {
                                        // If we managed to download the whole archive we don't need to
                                        // worry about zip64, it is so small that standard EOCD will do
                                        cdoff     = eocd->cdOffset;
                                        orgcdsz   = eocd->cdSize;
                                        orgcdcnt  = eocd->nbCdRec;
                                        buff = buff + cdoff;
                                        openstage = HaveCdRecords;
                                        continue;
                                      }
 
                                      // Let's see if it is ZIP64 (if yes, the EOCD will be preceded with ZIP64 EOCD locator)
                                      const char *zip64EocdlBlock = eocdBlock - ZIP64_EOCDL::zip64EocdlSize;
                                      // make sure there is enough data to assume there's a ZIP64 EOCD locator
                                      if( zip64EocdlBlock > buffer.get() )
                                      {
                                        uint32_t signature = to<uint32_t>( zip64EocdlBlock );
                                        if( signature == ZIP64_EOCDL::zip64EocdlSign )
                                        {
                                          buff = zip64EocdlBlock;
                                          openstage = HaveZip64EocdlBlk;
                                          continue;
                                        }
                                      }
 
                                      // It's not ZIP64, we already know where the CD records are
                                      // we need to read more data
                                      cdoff     = eocd->cdOffset;
                                      orgcdsz   = eocd->cdSize;
                                      orgcdcnt  = eocd->nbCdRec;
                                      rdoff     = eocd->cdOffset;
                                      rdsize    = eocd->cdSize;
                                      buffer.reset( new char[*rdsize] );
                                      rdbuff    = buffer.get();
                                      openstage = HaveCdRecords;
                                      log->Dump( ZipMsg, "[%p] Reading additional data at offset: %llu.",
                                                         (void*)this, (unsigned long long) *rdoff );
                                      Pipeline::Repeat(); break; // the break is really not needed ...
                                    }
 
                                    case HaveZip64EocdlBlk:
                                    {
                                      std::unique_ptr<ZIP64_EOCDL> eocdl( new ZIP64_EOCDL( buff ) );
                                      log->Dump( ZipMsg, "[%p] EOCDL record parsed: %s",
                                                         (void*)this, eocdl->ToString().c_str() );
 
                                      if( chunk.offset > eocdl->zip64EocdOffset )
                                      {
                                        // we need to read more data, adjust the read arguments
                                        rdsize = archsize - eocdl->zip64EocdOffset;
                                        rdoff  = eocdl->zip64EocdOffset;
                                        buffer.reset( new char[*rdsize] );
                                        rdbuff = buffer.get();
                                        openstage = HaveZip64EocdBlk;
                                        log->Dump( ZipMsg, "[%p] Reading additional data at offset: %llu.",
                                                           (void*)this, (unsigned long long) *rdoff );
                                        Pipeline::Repeat();
                                      }
 
                                      buff = buffer.get() + ( eocdl->zip64EocdOffset - chunk.offset );
                                      openstage = HaveZip64EocdBlk;
                                      continue;
                                    }
 
                                    case HaveZip64EocdBlk:
                                    {
                                      uint32_t signature = to<uint32_t>( buff );
                                      if( signature != ZIP64_EOCD::zip64EocdSign )
                                      {
                                        XRootDStatus error( stError, errDataError, 0,
                                                            "ZIP64 End-of-central-directory signature not found." );
                                        Pipeline::Stop( error );
                                      }
                                      zip64eocd.reset( new ZIP64_EOCD( buff ) );
                                      log->Dump( ZipMsg, "[%p] ZIP64EOCD record parsed: %s",
                                                         (void*)this, zip64eocd->ToString().c_str() );
 
                                      // now we can read the CD records, adjust the read arguments
                                      cdoff     = zip64eocd->cdOffset;
                                      orgcdsz   = zip64eocd->cdSize;
                                      orgcdcnt  = zip64eocd->nbCdRec;
                                      rdoff     = zip64eocd->cdOffset;
                                      rdsize    = zip64eocd->cdSize;
                                      buffer.reset( new char[*rdsize] );
                                      rdbuff    = buffer.get();
                                      openstage = HaveCdRecords;
                                      log->Dump( ZipMsg, "[%p] Reading additional data at offset: %llu.",
                                                         (void*)this, (unsigned long long) *rdoff );
                                      Pipeline::Repeat(); break; // the break is really not needed ...
                                    }
 
                                    case HaveCdRecords:
                                    {
                                      // make a copy of the original CDFH records
                                      orgcdbuf.reserve( orgcdsz );
                                      std::copy( buff, buff + orgcdsz, std::back_inserter( orgcdbuf ) );
                                      try
                                      {
                                        if( zip64eocd )
                                          std::tie( cdvec, cdmap ) = CDFH::Parse( buff, zip64eocd->cdSize, zip64eocd->nbCdRec );
                                        else
                                          std::tie( cdvec, cdmap ) = CDFH::Parse( buff, eocd->cdSize, eocd->nbCdRec );
                                        log->Dump( ZipMsg, "[%p] CD records parsed.", (void*)this );
                                        uint64_t sumCompSize = 0;
                                        for (auto it = cdvec.begin(); it != cdvec.end(); it++)
                                        {
                                          sumCompSize += (*it)->IsZIP64() ? (*it)->extra->compressedSize : (*it)->compressedSize;
                                          if ((*it)->offset > archsize || (*it)->offset + (*it)->compressedSize > archsize)
                                            throw bad_data();
                                        }
                                        if (sumCompSize > archsize)
                                          throw bad_data();
                                      }
                                      catch( const bad_data &ex )
                                      {
                                        XRootDStatus error( stError, errDataError, 0,
                                                                   "ZIP Central Directory corrupted." );
                                        Pipeline::Stop( error );
                                      }
                                      if( chunk.length != archsize ) buffer.reset();
                                      openstage = Done;
                                      cdexists  = true;
                                      break;
                                    }
 
                                    default: Pipeline::Stop( XRootDStatus( stError, errInvalidOp ) );
                                  }
 
                                  break;
                                }
                              }
                          | XrdCl::Final( [handler,log,url,this]( const XRootDStatus &status )
                              { // finalize the pipeline by calling the user callback
                                if( status.IsOK() )
                                  log->Debug( ZipMsg, "[%p] Opened a ZIP archive (%s): %s",
                                              (void*)this, url.c_str(), status.ToString().c_str() );
                                else
                                  log->Error( ZipMsg, "[%p] Failed to open a ZIP archive (%s): %s",
                                              (void*)this, url.c_str(), status.ToString().c_str() );
                                if( handler )
                                  handler->HandleResponse( make_status( status ), nullptr );
                              } );
 
    Async( std::move( open_archive ), timeout );
    return XRootDStatus();
  }

References XrdCl::Async(), XrdCl::Log::Debug(), XrdCl::Log::Dump(), XrdZip::EOCD::eocdBaseSize, XrdCl::errDataError, XrdCl::errInvalidOp, XrdCl::Log::Error(), XrdZip::EOCD::Find(), XrdCl::DefaultEnv::GetLog(), XrdCl::ResponseHandler::HandleResponse(), XrdZip::EOCD::maxCommentLength, XrdCl::Open(), XrdZip::CDFH::Parse(), XrdCl::Read(), XrdCl::Pipeline::Repeat(), XrdCl::stError, XrdCl::Pipeline::Stop(), XrdZip::to(), XrdZip::ZIP64_EOCDL::zip64EocdlSign, XrdZip::ZIP64_EOCDL::zip64EocdlSize, XrdZip::ZIP64_EOCD::zip64EocdSign, and XrdCl::ZipMsg.

Here is the call graph for this function:

OpenFile()

XRootDStatus XrdCl::ZipArchive::OpenFile	(	const std::string &	fn,
		OpenFlags::Flags	flags = OpenFlags::None,
		uint64_t	size = 0,
		uint32_t	crc32 = 0 )

Open a file within the ZIP Archive

Parameters

fn	: file name to be opened
flags	: open flags (either 'Read' or 'New | Write')
size	: file size (to be included in the LFH)
crc32	: file crc32 (to be included in the LFH)

Returns

: the status of the operation

Definition at line 489 of file XrdClZipArchive.cc.

  {
    if( !openfn.empty() || openstage != Done || !archive.IsOpen() )
      return XRootDStatus( stError, errInvalidOp );
 
    Log  *log = DefaultEnv::GetLog();
    auto  itr   = cdmap.find( fn );
    if( itr == cdmap.end() )
    {
      // the file does not exist in the archive so it only makes sense
      // if our user is opening for append
      if( flags & OpenFlags::New )
      {
        openfn = fn;
        lfh.reset( new LFH( fn, crc32, size, time( 0 ) ) );
        log->Dump( ZipMsg, "[%p] File %s opened for append.",
                           (void*)this, fn.c_str() );
        return XRootDStatus();
      }
      log->Dump( ZipMsg, "[%p] Open failed: %s not in the ZIP archive.",
                         (void*)this, fn.c_str() );
      return XRootDStatus( stError, errNotFound );
    }
 
    // the file name exist in the archive but our user wants to append
    // a file with the same name
    if( flags & OpenFlags::New )
    {
      log->Dump( ZipMsg, "[%p] Open failed: file exists %s, cannot append.",
                         (void*)this, fn.c_str() );
 
      return XRootDStatus( stError, errInvalidOp, EEXIST, "The file already exists in the ZIP archive." );
    }
 
    openfn = fn;
    log->Dump( ZipMsg, "[%p] File %s opened for reading.",
                       (void*)this, fn.c_str() );
    return XRootDStatus();
  }

References XrdCl::Log::Dump(), XrdCl::errInvalidOp, XrdCl::errNotFound, XrdCl::DefaultEnv::GetLog(), XrdCl::OpenFlags::New, XrdCl::stError, and XrdCl::ZipMsg.

Here is the call graph for this function:

PgRead()

XRootDStatus XrdCl::ZipArchive::PgRead ( uint64_t offset, uint32_t size, void * buffer, ResponseHandler * handler, uint16_t timeout = 0 )

inline

PgRead data from an open file

Parameters

offset	: offset within the file to read at
size	: number of bytes to be read
buffer	: the buffer for the data
handler	: user callback
timeout	: operation timeout

Returns

: the status of the operation

Definition at line 143 of file XrdClZipArchive.hh.

      {
        if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
        return PgReadFrom( openfn, offset, size, buffer, handler, timeout );
      }

References XrdCl::errInvalidOp, PgReadFrom(), and XrdCl::stError.

Here is the call graph for this function:

PgReadFrom()

XRootDStatus XrdCl::ZipArchive::PgReadFrom	(	const std::string &	fn,
		uint64_t	offset,
		uint32_t	size,
		void *	buffer,
		ResponseHandler *	handler,
		uint16_t	timeout = 0 )

PgRead data from a given file

Parameters

fn	: the name of the file from which we are going to read
offset	: offset within the file to read at
size	: number of bytes to be read
buffer	: the buffer for the data
handler	: user callback
timeout	: operation timeout

Returns

: the status of the operation

Definition at line 707 of file XrdClZipArchive.cc.

  {
    return ReadFromImpl<PageInfo>( *this, fn, offset, size, buffer, handler, timeout );
  }

References ReadFromImpl.

Referenced by PgRead().

Here is the call graph for this function:

Here is the caller graph for this function:

Read()

XRootDStatus XrdCl::ZipArchive::Read ( uint64_t offset, uint32_t size, void * buffer, ResponseHandler * handler, uint16_t timeout = 0 )

inline

Read data from an open file

Parameters

offset	: offset within the file to read at
size	: number of bytes to be read
buffer	: the buffer for the data
handler	: user callback
timeout	: operation timeout

Returns

: the status of the operation

Definition at line 122 of file XrdClZipArchive.hh.

      {
        if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
        return ReadFrom( openfn, offset, size, buffer, handler, timeout );
      }

References XrdCl::errInvalidOp, ReadFrom(), and XrdCl::stError.

Here is the call graph for this function:

ReadFrom()

XRootDStatus XrdCl::ZipArchive::ReadFrom	(	const std::string &	fn,
		uint64_t	offset,
		uint32_t	size,
		void *	buffer,
		ResponseHandler *	handler,
		uint16_t	timeout = 0 )

Read data from a given file

Parameters

fn	: the name of the file from which we are going to read
offset	: offset within the file to read at
size	: number of bytes to be read
buffer	: the buffer for the data
handler	: user callback
timeout	: operation timeout

Returns

: the status of the operation

Definition at line 694 of file XrdClZipArchive.cc.

  {
    return ReadFromImpl<ChunkInfo>( *this, fn, offset, size, buffer, handler, timeout );
  }

References ReadFromImpl.

Referenced by Read().

Here is the call graph for this function:

Here is the caller graph for this function:

SetProperty()

bool XrdCl::ZipArchive::SetProperty ( const std::string & name, const std::string & value )

inline

Set property on the underlying File object.

Definition at line 374 of file XrdClZipArchive.hh.

      {
        return archive.SetProperty( name, value );
      }

References XrdCl::NullRef< Response >::value.

Stat() [1/2]

XRootDStatus XrdCl::ZipArchive::Stat ( const std::string & fn, StatInfo *& info )

inline

Get stat info for given file

Parameters

fn	: the name of the file
info	: output parameter

Returns

: the status of the operation

Definition at line 242 of file XrdClZipArchive.hh.

      { // make sure archive has been opened and CD has been parsed
        if( openstage != Done )
          return XRootDStatus( stError, errInvalidOp );
        // make sure the file is part of the archive
        auto cditr = cdmap.find( fn );
        if( cditr == cdmap.end() )
          return XRootDStatus( stError, errNotFound );
        // create the result
        info = make_stat( fn );
        if (info) 
          return XRootDStatus();
        else // have difficult to access the openned archive.
          return XRootDStatus( stError, errNotFound );
      }

References XrdCl::errInvalidOp, XrdCl::errNotFound, and XrdCl::stError.

Stat() [2/2]

XRootDStatus XrdCl::ZipArchive::Stat ( StatInfo *& info )

inline

Get stat info for an open file

Parameters

info	: output parameter

Returns

: the status of the operation

Definition at line 264 of file XrdClZipArchive.hh.

      {
        if( openfn.empty() )
          return XRootDStatus( stError, errInvalidOp );
        return Stat( openfn, info );
      }

References XrdCl::errInvalidOp, Stat, and XrdCl::stError.

UpdateMetadata()

XRootDStatus XrdCl::ZipArchive::UpdateMetadata

(

uint32_t

crc32

)

Update the metadata of the currently open file

Parameters

crc32

: the crc32 checksum

Returns

: the status of the operation

Definition at line 849 of file XrdClZipArchive.cc.

  {
    if( openstage != Done || openfn.empty() )
      return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." );
 
    //---------------------------------------------------------------------
    // Firstly, update the crc32 in the central directory
    //---------------------------------------------------------------------
    auto itr = cdmap.find( openfn );
    if( itr == cdmap.end() )
      return XRootDStatus( stError, errInvalidOp );
    cdvec[itr->second]->ZCRC32 = crc32;
 
    //---------------------------------------------------------------------
    // Secondly, update the crc32 in the LFH and mark it as needing
    // overwriting
    //---------------------------------------------------------------------
    auto itr2 = newfiles.find( openfn );
    if( itr2 == newfiles.end() )
      return XRootDStatus( stError, errInvalidOp );
    itr2->second.lfh->ZCRC32 = crc32;
 
    return XRootDStatus();
  }

References XrdCl::errInvalidOp, and XrdCl::stError.

Write()

XRootDStatus XrdCl::ZipArchive::Write ( uint32_t size, const void * buffer, ResponseHandler * handler, uint16_t timeout = 0 )

inline

Append data to a new file

Parameters

size	: number of bytes to be appended
buffer	: the buffer with the data to be appended
handler	: user callback
timeout	: operation timeout

Returns

: the status of the operation

Definition at line 198 of file XrdClZipArchive.hh.

      {
        if( openstage != Done || openfn.empty() )
          return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." );
 
        return WriteImpl( size, buffer, handler, timeout );
      }

References XrdCl::errInvalidOp, and XrdCl::stError.

Friends And Related Symbol Documentation

::MicroTest

friend class ::MicroTest

friend

Definition at line 66 of file XrdClZipArchive.hh.

::XrdEcTests

friend class ::XrdEcTests

friend

Definition at line 67 of file XrdClZipArchive.hh.

References ZipArchive(), and XrdCl::OpenFlags::None.

ReadFromImpl

template<typename RSP>

XRootDStatus ReadFromImpl ( ZipArchive & me, const std::string & fn, uint64_t relativeOffset, uint32_t size, void * usrbuff, ResponseHandler * usrHandler, uint16_t timeout )

friend

Definition at line 44 of file XrdClZipArchive.cc.

  {
    if( me.openstage != ZipArchive::Done || !me.archive.IsOpen() )
      return XRootDStatus( stError, errInvalidOp );
 
    Log *log = DefaultEnv::GetLog();
 
    auto cditr = me.cdmap.find( fn );
    if( cditr == me.cdmap.end() )
      return XRootDStatus( stError, errNotFound,
                           errNotFound, "File not found." );
 
    CDFH *cdfh = me.cdvec[cditr->second].get();
 
    // check if the file is compressed, for now we only support uncompressed and inflate/deflate compression
    if( cdfh->compressionMethod != 0 && cdfh->compressionMethod != Z_DEFLATED )
      return XRootDStatus( stError, errNotSupported,
                           0, "The compression algorithm is not supported!" );
 
    // Now the problem is that at the beginning of our
    // file there is the Local-file-header, which size
    // is not known because of the variable size 'extra'
    // field, so we need to know the offset of the next
    // record and shift it by the file size.
    // The next record is either the next LFH (next file)
    // or the start of the Central-directory.
    uint64_t cdOffset = me.zip64eocd ? me.zip64eocd->cdOffset : me.eocd->cdOffset;
    uint64_t nextRecordOffset = ( cditr->second + 1 < me.cdvec.size() ) ?
                                CDFH::GetOffset( *me.cdvec[cditr->second + 1] ) : cdOffset;
    uint64_t filesize = cdfh->compressedSize;
    if( filesize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
      filesize = cdfh->extra->compressedSize;
    uint16_t descsize = cdfh->HasDataDescriptor() ?
                        DataDescriptor::GetSize( cdfh->IsZIP64() ) : 0;
    uint64_t fileoff  = nextRecordOffset - filesize - descsize;
    uint64_t offset   = fileoff + relativeOffset;
    uint64_t uncompressedSize = cdfh->uncompressedSize;
    if( uncompressedSize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
      uncompressedSize = cdfh->extra->uncompressedSize;
    uint64_t sizeTillEnd = relativeOffset > uncompressedSize ?
                           0 : uncompressedSize - relativeOffset;
    if( size > sizeTillEnd ) size = sizeTillEnd;
 
    // if it is a compressed file use ZIP cache to read from the file
    if( cdfh->compressionMethod == Z_DEFLATED )
    {
      log->Dump( ZipMsg, "[%p] Reading compressed data.", (void*)&me );
      // check if respective ZIP cache exists
      bool empty = me.zipcache.find( fn ) == me.zipcache.end();
      // if the entry does not exist, it will be created using
      // default constructor
      ZipCache &cache = me.zipcache[fn];
 
      if( relativeOffset > uncompressedSize )
      {
        // we are reading past the end of file,
        // we can serve the request right away!
        RSP *r = new RSP( relativeOffset, 0, usrbuff );
        AnyObject *rsp = new AnyObject();
        rsp->Set( r );
        usrHandler->HandleResponse( new XRootDStatus(), rsp );
        return XRootDStatus();
      }
 
      uint32_t sizereq = size;
      if( relativeOffset + size > uncompressedSize )
        sizereq = uncompressedSize - relativeOffset;
      cache.QueueReq( relativeOffset, sizereq, usrbuff, usrHandler );
 
      // if we have the whole ZIP archive we can populate the cache
      // straight away
      if( empty && me.buffer)
      {
        auto begin = me.buffer.get() + fileoff;
        auto end   = begin + filesize ;
        buffer_t buff( begin, end );
        cache.QueueRsp( XRootDStatus(), 0, std::move( buff ) );
        return XRootDStatus();
      }
 
      // if we don't have the data we need to issue a remote read
      if( !me.buffer )
      {
        if( relativeOffset > filesize ) return XRootDStatus(); // there's nothing to do,
                                                                           // we already have all the data locally
        uint32_t rdsize = size;
        // check if this is the last read (we reached the end of
        // file from user perspective)
        if( relativeOffset + size >= uncompressedSize )
        {
          // if yes, make sure we readout all the compressed data
          // Note: In a patological case the compressed size may
          //       be greater than the uncompressed size
          rdsize = filesize > relativeOffset ?
                   filesize - relativeOffset :
                   0;
        }
        // make sure we are not reading past the end of
        // compressed data
        if( relativeOffset + size > filesize )
          rdsize = filesize - relativeOffset;
 
 
        // now read the data ...
        auto rdbuff = std::make_shared<ZipCache::buffer_t>( rdsize );
        Pipeline p = XrdCl::RdWithRsp<RSP>( me.archive, offset, rdbuff->size(), rdbuff->data() ) >>
                       [relativeOffset, rdbuff, &cache, &me]( XRootDStatus &st, RSP &rsp )
                       {
                         Log *log = DefaultEnv::GetLog();
                         log->Dump( ZipMsg, "[%p] Read %u bytes of remote data at offset %llu.",
                                            (void*)&me, rsp.GetLength(), (unsigned long long) rsp.GetOffset() );
                         cache.QueueRsp( st, relativeOffset, std::move( *rdbuff ) );
                       };
        Async( std::move( p ), timeout );
      }
 
      return XRootDStatus();
    }
 
    // check if we have the whole file in our local buffer
    if( me.buffer || size == 0 )
    {
      if( size )
      {
        memcpy( usrbuff, me.buffer.get() + offset, size );
        log->Dump( ZipMsg, "[%p] Serving read from local cache.", (void*)&me );
      }
 
      if( usrHandler )
      {
        XRootDStatus *st  = ZipArchive::make_status();
        RSP          *rsp = new RSP( relativeOffset, size, usrbuff );
        ZipArchive::Schedule( usrHandler, st, rsp );
      }
      return XRootDStatus();
    }
 
    Pipeline p = XrdCl::RdWithRsp<RSP>( me.archive, offset, size, usrbuff ) >>
                   [=, &me]( XRootDStatus &st, RSP &r )
                   {
                     log->Dump( ZipMsg, "[%p] Read %u bytes of remote data at "
                                        "offset %llu.", (void*)&me, r.GetLength(), (unsigned long long) r.GetOffset() );
                     if( usrHandler )
                     {
                       XRootDStatus *status = ZipArchive::make_status( st );
                       RSP          *rsp    = nullptr;
                       if( st.IsOK() )
                         rsp = new RSP( relativeOffset, r.GetLength(), r.GetBuffer() );
                       usrHandler->HandleResponse( status, ZipArchive::PkgRsp( rsp ) );
                     }
                   };
    Async( std::move( p ), timeout );
    return XRootDStatus();
  }

References ZipArchive(), XrdCl::Async(), XrdZip::CDFH::compressedSize, XrdZip::CDFH::compressionMethod, XrdCl::Log::Dump(), XrdCl::errInvalidOp, XrdCl::errNotFound, XrdCl::errNotSupported, XrdZip::CDFH::extra, XrdCl::DefaultEnv::GetLog(), GetOffset(), XrdZip::CDFH::GetOffset(), XrdZip::DataDescriptor::GetSize(), XrdCl::ResponseHandler::HandleResponse(), XrdZip::CDFH::HasDataDescriptor(), XrdCl::Status::IsOK(), XrdCl::File::IsOpen(), XrdZip::CDFH::IsZIP64(), XrdCl::ZipCache::QueueReq(), XrdCl::ZipCache::QueueRsp(), XrdCl::RdWithRsp(), XrdCl::AnyObject::Set(), XrdCl::stError, XrdZip::CDFH::uncompressedSize, and XrdCl::ZipMsg.

Referenced by PgReadFrom(), and ReadFrom().

XrdEc::OpenOnlyImpl

template<bool>

friend class XrdEc::OpenOnlyImpl

friend

Definition at line 65 of file XrdClZipArchive.hh.

XrdEc::Reader

friend class XrdEc::Reader

friend

Definition at line 63 of file XrdClZipArchive.hh.

XrdEc::StrmWriter

friend class XrdEc::StrmWriter

friend

Definition at line 62 of file XrdClZipArchive.hh.

---

The documentation for this class was generated from the following files:

• XrdClZipArchive.hh
• XrdClZipArchive.cc