xgboost
io.h
Go to the documentation of this file.
1 
8 #ifndef XGBOOST_COMMON_IO_H_
9 #define XGBOOST_COMMON_IO_H_
10 
11 #include <dmlc/io.h>
12 #include <rabit/rabit.h>
13 #include <string>
14 #include <cstring>
15 
16 #include "common.h"
17 
18 namespace xgboost {
19 namespace common {
22 
27 class PeekableInStream : public dmlc::Stream {
28  public:
29  explicit PeekableInStream(dmlc::Stream* strm)
30  : strm_(strm), buffer_ptr_(0) {}
31 
32  size_t Read(void* dptr, size_t size) override;
33  virtual size_t PeekRead(void* dptr, size_t size);
34 
35  void Write(const void*, size_t) override {
36  LOG(FATAL) << "Not implemented";
37  }
38 
39  private:
41  dmlc::Stream *strm_;
43  size_t buffer_ptr_;
45  std::string buffer_;
46 };
53  public:
54  explicit FixedSizeStream(PeekableInStream* stream);
55  ~FixedSizeStream() override = default;
56 
57  size_t Read(void* dptr, size_t size) override;
58  size_t PeekRead(void* dptr, size_t size) override;
59  size_t Size() const { return buffer_.size(); }
60  size_t Tell() const { return pointer_; }
61  void Seek(size_t pos);
62 
63  void Write(const void*, size_t) override {
64  LOG(FATAL) << "Not implemented";
65  }
66 
71  void Take(std::string* out);
72 
73  private:
74  size_t pointer_;
75  std::string buffer_;
76 };
77 
87 std::string LoadSequentialFile(std::string uri, bool stream = false);
88 
96 std::string FileExtension(std::string fname, bool lower = true);
97 
101 inline std::string ReadAll(dmlc::Stream* fi, PeekableInStream* fp) {
102  std::string buffer;
103  if (auto fixed_size = dynamic_cast<common::MemoryFixSizeBuffer*>(fi)) {
104  fixed_size->Seek(common::MemoryFixSizeBuffer::kSeekEnd);
105  size_t size = fixed_size->Tell();
106  buffer.resize(size);
107  fixed_size->Seek(0);
108  CHECK_EQ(fixed_size->Read(&buffer[0], size), size);
109  } else {
110  FixedSizeStream{fp}.Take(&buffer);
111  }
112  return buffer;
113 }
114 } // namespace common
115 } // namespace xgboost
116 #endif // XGBOOST_COMMON_IO_H_
xgboost::common::FixedSizeStream::PeekRead
size_t PeekRead(void *dptr, size_t size) override
xgboost::common::ReadAll
std::string ReadAll(dmlc::Stream *fi, PeekableInStream *fp)
Read the whole buffer from dmlc stream.
Definition: io.h:101
xgboost::common::PeekableInStream::Read
size_t Read(void *dptr, size_t size) override
xgboost::common::MemoryFixSizeBuffer
rabit::utils::MemoryFixSizeBuffer MemoryFixSizeBuffer
Definition: io.h:20
xgboost::common::FixedSizeStream::Take
void Take(std::string *out)
Take the buffer from ‘FixedSizeStream’. The one in ‘FixedSizeStream’ will be cleared out.
xgboost::common::LoadSequentialFile
std::string LoadSequentialFile(std::string uri, bool stream=false)
Helper function for loading consecutive file to avoid dmlc Stream when possible.
xgboost::common::FixedSizeStream::FixedSizeStream
FixedSizeStream(PeekableInStream *stream)
xgboost::common::FixedSizeStream::Write
void Write(const void *, size_t) override
Definition: io.h:63
xgboost::common::FileExtension
std::string FileExtension(std::string fname, bool lower=true)
Get file extension from file name.
xgboost::common::FixedSizeStream::Tell
size_t Tell() const
Definition: io.h:60
xgboost::common::PeekableInStream::PeekRead
virtual size_t PeekRead(void *dptr, size_t size)
xgboost::common::FixedSizeStream::~FixedSizeStream
~FixedSizeStream() override=default
xgboost::common::PeekableInStream
Input stream that support additional PeekRead operation, besides read.
Definition: io.h:27
xgboost::common::FixedSizeStream
A simple class used to consume ‘dmlc::Stream’ all at once.
Definition: io.h:52
xgboost::common::FixedSizeStream::Size
size_t Size() const
Definition: io.h:59
common.h
Common utilities.
xgboost::common::PeekableInStream::Write
void Write(const void *, size_t) override
Definition: io.h:35
xgboost::common::MemoryBufferStream
rabit::utils::MemoryBufferStream MemoryBufferStream
Definition: io.h:21
xgboost::common::FixedSizeStream::Seek
void Seek(size_t pos)
xgboost::common::PeekableInStream::PeekableInStream
PeekableInStream(dmlc::Stream *strm)
Definition: io.h:29
xgboost::common::FixedSizeStream::Read
size_t Read(void *dptr, size_t size) override
xgboost
namespace of xgboost
Definition: base.h:110