xgboost

io.h
Go to the documentation of this file.
1
8 #ifndef XGBOOST_COMMON_IO_H_
9 #define XGBOOST_COMMON_IO_H_
10
11 #include <dmlc/io.h>
12 #include <rabit/rabit.h>
13 #include <string>
14 #include <cstring>
15
16 #include "common.h"
17
18 namespace xgboost {
19 namespace common {
20 using MemoryFixSizeBuffer = rabit::utils::MemoryFixSizeBuffer;
21 using MemoryBufferStream = rabit::utils::MemoryBufferStream;
22
27 class PeekableInStream : public dmlc::Stream {
28 public:
29 explicit PeekableInStream(dmlc::Stream* strm)
30 : strm_(strm), buffer_ptr_(0) {}
31
32 size_t Read(void* dptr, size_t size) override;
33 virtual size_t PeekRead(void* dptr, size_t size);
34
35 void Write(const void*, size_t) override {
36 LOG(FATAL) << "Not implemented";
37 }
38
39 private:
41 dmlc::Stream *strm_;
43 size_t buffer_ptr_;
45 std::string buffer_;
46 };
52 class FixedSizeStream : public PeekableInStream {
53 public:
54 explicit FixedSizeStream(PeekableInStream* stream);
55 ~FixedSizeStream() override = default;
56
57 size_t Read(void* dptr, size_t size) override;
58 size_t PeekRead(void* dptr, size_t size) override;
59 size_t Size() const { return buffer_.size(); }
60 size_t Tell() const { return pointer_; }
61 void Seek(size_t pos);
62
63 void Write(const void*, size_t) override {
64 LOG(FATAL) << "Not implemented";
65 }
66
71 void Take(std::string* out);
72
73 private:
74 size_t pointer_;
75 std::string buffer_;
76 };
77
87 std::string LoadSequentialFile(std::string uri, bool stream = false);
88
96 std::string FileExtension(std::string fname, bool lower = true);
97
101 inline std::string ReadAll(dmlc::Stream* fi, PeekableInStream* fp) {
102 std::string buffer;
103 if (auto fixed_size = dynamic_cast<common::MemoryFixSizeBuffer*>(fi)) {
104 fixed_size->Seek(common::MemoryFixSizeBuffer::kSeekEnd);
105 size_t size = fixed_size->Tell();
106 buffer.resize(size);
107 fixed_size->Seek(0);
108 CHECK_EQ(fixed_size->Read(&buffer[0], size), size);
109 } else {
110 FixedSizeStream{fp}.Take(&buffer);
111 }
112 return buffer;
113 }
114 } // namespace common
115 } // namespace xgboost
116 #endif // XGBOOST_COMMON_IO_H_
xgboost::common::FixedSizeStream::PeekRead
size_t PeekRead(void *dptr, size_t size) override
xgboost::common::ReadAll
std::string ReadAll(dmlc::Stream *fi, PeekableInStream *fp)
Read the whole buffer from dmlc stream.
Definition: io.h:101
xgboost::common::PeekableInStream::Read
size_t Read(void *dptr, size_t size) override
xgboost::common::MemoryFixSizeBuffer
rabit::utils::MemoryFixSizeBuffer MemoryFixSizeBuffer
Definition: io.h:20
xgboost::common::FixedSizeStream::Take
void Take(std::string *out)
Take the buffer from ‘FixedSizeStream’. The one in ‘FixedSizeStream’ will be cleared out.
xgboost::common::LoadSequentialFile
std::string LoadSequentialFile(std::string uri, bool stream=false)
Helper function for loading consecutive file to avoid dmlc Stream when possible.
xgboost::common::FixedSizeStream::FixedSizeStream
FixedSizeStream(PeekableInStream *stream)
xgboost::common::FixedSizeStream::Write
void Write(const void *, size_t) override
Definition: io.h:63
xgboost::common::FileExtension
std::string FileExtension(std::string fname, bool lower=true)
Get file extension from file name.
xgboost::common::FixedSizeStream::Tell
size_t Tell() const
Definition: io.h:60
xgboost::common::PeekableInStream::PeekRead
virtual size_t PeekRead(void *dptr, size_t size)
xgboost::common::FixedSizeStream::~FixedSizeStream
~FixedSizeStream() override=default
xgboost::common::PeekableInStream
Input stream that support additional PeekRead operation, besides read.
Definition: io.h:27
xgboost::common::FixedSizeStream
A simple class used to consume ‘dmlc::Stream’ all at once.
Definition: io.h:52
xgboost::common::FixedSizeStream::Size
size_t Size() const
Definition: io.h:59
common.h
Common utilities.
xgboost::common::PeekableInStream::Write
void Write(const void *, size_t) override
Definition: io.h:35
xgboost::common::MemoryBufferStream
rabit::utils::MemoryBufferStream MemoryBufferStream
Definition: io.h:21
xgboost::common::FixedSizeStream::Seek
void Seek(size_t pos)
xgboost::common::PeekableInStream::PeekableInStream
PeekableInStream(dmlc::Stream *strm)
Definition: io.h:29
xgboost::common::FixedSizeStream::Read
size_t Read(void *dptr, size_t size) override
xgboost
namespace of xgboost
Definition: base.h:110