xgboost
io.h
Go to the documentation of this file.
1 
8 #ifndef XGBOOST_COMMON_IO_H_
9 #define XGBOOST_COMMON_IO_H_
10 
11 #include <dmlc/io.h>
12 #include <rabit/rabit.h>
13 #include <string>
14 #include <cstring>
15 
16 #include "common.h"
17 
18 namespace xgboost {
19 namespace common {
22 
27 class PeekableInStream : public dmlc::Stream {
28  public:
29  explicit PeekableInStream(dmlc::Stream* strm)
30  : strm_(strm), buffer_ptr_(0) {}
31 
32  size_t Read(void* dptr, size_t size) override;
33  virtual size_t PeekRead(void* dptr, size_t size);
34 
35  void Write(const void*, size_t) override {
36  LOG(FATAL) << "Not implemented";
37  }
38 
39  private:
41  dmlc::Stream *strm_;
43  size_t buffer_ptr_;
45  std::string buffer_;
46 };
53  public:
54  explicit FixedSizeStream(PeekableInStream* stream);
55  ~FixedSizeStream() override = default;
56 
57  size_t Read(void* dptr, size_t size) override;
58  size_t PeekRead(void* dptr, size_t size) override;
59  size_t Size() const { return buffer_.size(); }
60  size_t Tell() const { return pointer_; }
61  void Seek(size_t pos);
62 
63  void Write(const void*, size_t) override {
64  LOG(FATAL) << "Not implemented";
65  }
66 
71  void Take(std::string* out);
72 
73  private:
74  size_t pointer_;
75  std::string buffer_;
76 };
77 
87 std::string LoadSequentialFile(std::string uri, bool stream = false);
88 
89 inline std::string FileExtension(std::string const& fname) {
90  auto splited = Split(fname, '.');
91  if (splited.size() > 1) {
92  return splited.back();
93  } else {
94  return "";
95  }
96 }
97 
98 } // namespace common
99 } // namespace xgboost
100 #endif // XGBOOST_COMMON_IO_H_
xgboost::common::FixedSizeStream::PeekRead
size_t PeekRead(void *dptr, size_t size) override
xgboost::common::PeekableInStream::Read
size_t Read(void *dptr, size_t size) override
xgboost::common::MemoryFixSizeBuffer
rabit::utils::MemoryFixSizeBuffer MemoryFixSizeBuffer
Definition: io.h:20
xgboost::common::FixedSizeStream::Take
void Take(std::string *out)
Take the buffer from ‘FixedSizeStream’. The one in ‘FixedSizeStream’ will be cleared out.
xgboost::common::FileExtension
std::string FileExtension(std::string const &fname)
Definition: io.h:89
xgboost::common::LoadSequentialFile
std::string LoadSequentialFile(std::string uri, bool stream=false)
Helper function for loading consecutive file to avoid dmlc Stream when possible.
xgboost::common::FixedSizeStream::FixedSizeStream
FixedSizeStream(PeekableInStream *stream)
xgboost::common::FixedSizeStream::Write
void Write(const void *, size_t) override
Definition: io.h:63
xgboost::common::FixedSizeStream::Tell
size_t Tell() const
Definition: io.h:60
xgboost::common::PeekableInStream::PeekRead
virtual size_t PeekRead(void *dptr, size_t size)
xgboost::common::FixedSizeStream::~FixedSizeStream
~FixedSizeStream() override=default
xgboost::common::PeekableInStream
Input stream that support additional PeekRead operation, besides read.
Definition: io.h:27
xgboost::common::FixedSizeStream
A simple class used to consume ‘dmlc::Stream’ all at once.
Definition: io.h:52
xgboost::common::FixedSizeStream::Size
size_t Size() const
Definition: io.h:59
common.h
Common utilities.
xgboost::common::PeekableInStream::Write
void Write(const void *, size_t) override
Definition: io.h:35
xgboost::common::MemoryBufferStream
rabit::utils::MemoryBufferStream MemoryBufferStream
Definition: io.h:21
xgboost::common::FixedSizeStream::Seek
void Seek(size_t pos)
xgboost::common::PeekableInStream::PeekableInStream
PeekableInStream(dmlc::Stream *strm)
Definition: io.h:29
xgboost::common::FixedSizeStream::Read
size_t Read(void *dptr, size_t size) override
xgboost
namespace of xgboost
Definition: base.h:110
xgboost::common::Split
std::vector< std::string > Split(const std::string &s, char delim)
Split a string by delimiter.
Definition: common.h:61