xgboost
io.h
Go to the documentation of this file.
1 
8 #ifndef XGBOOST_COMMON_IO_H_
9 #define XGBOOST_COMMON_IO_H_
10 
11 #include <dmlc/io.h>
12 #include <rabit/rabit.h>
13 #include <string>
14 #include <cstring>
15 
16 #include "common.h"
17 
18 namespace xgboost {
19 namespace common {
22 
27 class PeekableInStream : public dmlc::Stream {
28  public:
29  explicit PeekableInStream(dmlc::Stream* strm)
30  : strm_(strm), buffer_ptr_(0) {}
31 
32  size_t Read(void* dptr, size_t size) override;
33  virtual size_t PeekRead(void* dptr, size_t size);
34 
35  void Write(const void* dptr, size_t size) override {
36  LOG(FATAL) << "Not implemented";
37  }
38 
39  private:
41  dmlc::Stream *strm_;
43  size_t buffer_ptr_;
45  std::string buffer_;
46 };
53  public:
54  explicit FixedSizeStream(PeekableInStream* stream);
55  ~FixedSizeStream() override = default;
56 
57  size_t Read(void* dptr, size_t size) override;
58  size_t PeekRead(void* dptr, size_t size) override;
59  size_t Size() const { return buffer_.size(); }
60  size_t Tell() const { return pointer_; }
61  void Seek(size_t pos);
62 
63  void Write(const void* dptr, size_t size) override {
64  LOG(FATAL) << "Not implemented";
65  }
66 
71  void Take(std::string* out);
72 
73  private:
74  size_t pointer_;
75  std::string buffer_;
76 };
77 
87 std::string LoadSequentialFile(std::string uri, bool stream = false);
88 
89 inline std::string FileExtension(std::string const& fname) {
90  auto splited = Split(fname, '.');
91  if (splited.size() > 1) {
92  return splited.back();
93  } else {
94  return "";
95  }
96 }
97 
98 } // namespace common
99 } // namespace xgboost
100 #endif // XGBOOST_COMMON_IO_H_
virtual size_t PeekRead(void *dptr, size_t size)
rabit::utils::MemoryBufferStream MemoryBufferStream
Definition: io.h:21
std::string LoadSequentialFile(std::string uri, bool stream=false)
Helper function for loading consecutive file to avoid dmlc Stream when possible.
A simple class used to consume `dmlc::Stream&#39; all at once.
Definition: io.h:52
size_t Read(void *dptr, size_t size) override
size_t Size() const
Definition: io.h:59
void Write(const void *dptr, size_t size) override
Definition: io.h:35
std::vector< std::string > Split(const std::string &s, char delim)
Split a string by delimiter.
Definition: common.h:57
size_t Tell() const
Definition: io.h:60
void Write(const void *dptr, size_t size) override
Definition: io.h:63
Input stream that support additional PeekRead operation, besides read.
Definition: io.h:27
namespace of xgboost
Definition: base.h:102
std::string FileExtension(std::string const &fname)
Definition: io.h:89
Common utilities.
rabit::utils::MemoryFixSizeBuffer MemoryFixSizeBuffer
Definition: io.h:20
PeekableInStream(dmlc::Stream *strm)
Definition: io.h:29