xgboost

json_io.h
Go to the documentation of this file.
1
4 #ifndef XGBOOST_JSON_IO_H_
5 #define XGBOOST_JSON_IO_H_
6 #include <dmlc/endian.h>
7 #include <xgboost/base.h>
8 #include <xgboost/json.h>
9
10 #include <cinttypes>
11 #include <limits>
12 #include <map>
13 #include <memory>
14 #include <sstream>
15 #include <string>
16 #include <utility>
17 #include <vector>
18
19 namespace xgboost {
20 /*
21 * \brief A json reader, currently error checking and utf-8 is not fully supported.
22 */
23 class JsonReader {
24 protected:
25 size_t constexpr static kMaxNumLength =
26 std::numeric_limits<double>::max_digits10 + 1;
27
28 struct SourceLocation {
29 private:
30 size_t pos_ { 0 }; // current position in raw_str_
31
32 public:
33 SourceLocation() = default;
34 size_t Pos() const { return pos_; }
35
36 void Forward() {
37 pos_++;
38 }
39 void Forward(uint32_t n) {
40 pos_ += n;
41 }
42 } cursor_;
43
44 StringView raw_str_;
45
46 protected:
47 void SkipSpaces();
48
49 char GetNextChar() {
50 if (XGBOOST_EXPECT((cursor_.Pos() == raw_str_.size()), false)) {
51 return -1;
52 }
53 char ch = raw_str_[cursor_.Pos()];
54 cursor_.Forward();
55 return ch;
56 }
57
58 char PeekNextChar() {
59 if (cursor_.Pos() == raw_str_.size()) {
60 return -1;
61 }
62 char ch = raw_str_[cursor_.Pos()];
63 return ch;
64 }
65
66 /* \brief Skip spaces and consume next character. */
67 char GetNextNonSpaceChar() {
68 SkipSpaces();
69 return GetNextChar();
70 }
71 /* \brief Consume next character without first skipping empty space, throw when the next
72 * character is not the expected one.
73 */
74 char GetConsecutiveChar(char expected_char) {
75 char result = GetNextChar();
76 if (XGBOOST_EXPECT(result != expected_char, false)) { Expect(expected_char, result); }
77 return result;
78 }
79
80 void Error(std::string msg) const;
81
82 // Report expected character
83 void Expect(char c, char got) {
84 std::string msg = "Expecting: \"";
85 msg += c;
86 msg += "\", got: \"";
87 if (got == EOF) {
88 msg += "EOF\"";
89 } else if (got == 0) {
90 msg += "\\0\"";
91 } else {
92 msg += (got <= 127 ? std::string{got} : std::to_string(got)) + " \""; // NOLINT
93 }
94 Error(msg);
95 }
96
97 virtual Json ParseString();
98 virtual Json ParseObject();
99 virtual Json ParseArray();
100 virtual Json ParseNumber();
101 virtual Json ParseBoolean();
102 virtual Json ParseNull();
103
104 Json Parse();
105
106 public:
107 explicit JsonReader(StringView str) :
108 raw_str_{str} {}
109
110 virtual ~JsonReader() = default;
111
112 virtual Json Load();
113 };
114
115 class JsonWriter {
116 template <typename T, std::enable_if_t<!std::is_same<Json, T>::value>* = nullptr>
117 void Save(T const& v) {
118 this->Save(Json{v});
119 }
120 template <typename Array, typename Fn>
121 void WriteArray(Array const* arr, Fn&& fn) {
122 stream_->emplace_back('[');
123 auto const& vec = arr->GetArray();
124 size_t size = vec.size();
125 for (size_t i = 0; i < size; ++i) {
126 auto const& value = vec[i];
127 this->Save(fn(value));
128 if (i != size - 1) {
129 stream_->emplace_back(',');
130 }
131 }
132 stream_->emplace_back(']');
133 }
134
135 protected:
136 std::vector<char>* stream_;
137
138 public:
139 explicit JsonWriter(std::vector<char>* stream) : stream_{stream} {}
140
141 virtual ~JsonWriter() = default;
142
143 virtual void Save(Json json);
144
145 virtual void Visit(JsonArray const* arr);
146 virtual void Visit(F32Array const* arr);
147 virtual void Visit(U8Array const* arr);
148 virtual void Visit(I32Array const* arr);
149 virtual void Visit(I64Array const* arr);
150 virtual void Visit(JsonObject const* obj);
151 virtual void Visit(JsonNumber const* num);
152 virtual void Visit(JsonInteger const* num);
153 virtual void Visit(JsonNull const* null);
154 virtual void Visit(JsonString const* str);
155 virtual void Visit(JsonBoolean const* boolean);
156 };
157
158 #if defined(__GLIBC__)
159 template <typename T>
160 T BuiltinBSwap(T v);
161
162 template <>
163 inline uint16_t BuiltinBSwap(uint16_t v) {
164 return __builtin_bswap16(v);
165 }
166
167 template <>
168 inline uint32_t BuiltinBSwap(uint32_t v) {
169 return __builtin_bswap32(v);
170 }
171
172 template <>
173 inline uint64_t BuiltinBSwap(uint64_t v) {
174 return __builtin_bswap64(v);
175 }
176 #else
177 template <typename T>
178 T BuiltinBSwap(T v) {
179 dmlc::ByteSwap(&v, sizeof(v), 1);
180 return v;
181 }
182 #endif // defined(__GLIBC__)
183
184 template <typename T, std::enable_if_t<sizeof(T) == 1>* = nullptr>
185 inline T ToBigEndian(T v) {
186 return v;
187 }
188
189 template <typename T, std::enable_if_t<sizeof(T) != 1>* = nullptr>
190 inline T ToBigEndian(T v) {
191 static_assert(std::is_pod<T>::value, "Only pod is supported.");
192 #if DMLC_LITTLE_ENDIAN
193 auto constexpr kS = sizeof(T);
194 std::conditional_t<kS == 2, uint16_t, std::conditional_t<kS == 4, uint32_t, uint64_t>> u;
195 std::memcpy(&u, &v, sizeof(u));
196 u = BuiltinBSwap(u);
197 std::memcpy(&v, &u, sizeof(u));
198 #endif // DMLC_LITTLE_ENDIAN
199 return v;
200 }
201
205 class UBJReader : public JsonReader {
206 Json Parse();
207
208 template <typename T>
209 T ReadStream() {
210 auto ptr = this->raw_str_.c_str() + cursor_.Pos();
211 T v{0};
212 std::memcpy(&v, ptr, sizeof(v));
213 cursor_.Forward(sizeof(v));
214 return v;
215 }
216
217 template <typename T>
218 T ReadPrimitive() {
219 auto v = ReadStream<T>();
220 v = ToBigEndian(v);
221 return v;
222 }
223
224 template <typename TypedArray>
225 auto ParseTypedArray(int64_t n) {
226 TypedArray results{static_cast<size_t>(n)};
227 for (int64_t i = 0; i < n; ++i) {
228 auto v = this->ReadPrimitive<typename TypedArray::Type>();
229 results.Set(i, v);
230 }
231 return Json{std::move(results)};
232 }
233
234 std::string DecodeStr();
235
236 Json ParseArray() override;
237 Json ParseObject() override;
238
239 public:
240 using JsonReader::JsonReader;
241 Json Load() override;
242 };
243
247 class UBJWriter : public JsonWriter {
248 void Visit(JsonArray const* arr) override;
249 void Visit(F32Array const* arr) override;
250 void Visit(U8Array const* arr) override;
251 void Visit(I32Array const* arr) override;
252 void Visit(I64Array const* arr) override;
253 void Visit(JsonObject const* obj) override;
254 void Visit(JsonNumber const* num) override;
255 void Visit(JsonInteger const* num) override;
256 void Visit(JsonNull const* null) override;
257 void Visit(JsonString const* str) override;
258 void Visit(JsonBoolean const* boolean) override;
259
260 public:
261 using JsonWriter::JsonWriter;
262 void Save(Json json) override;
263 };
264 } // namespace xgboost
265
266 #endif // XGBOOST_JSON_IO_H_
xgboost::JsonBoolean
Describes both true and false.
Definition: json.h:307
xgboost::JsonWriter::Visit
virtual void Visit(JsonArray const *arr)
xgboost::JsonNumber
Definition: json.h:214
xgboost::JsonReader::Parse
Json Parse()
xgboost::U8Array
JsonTypedArray< uint8_t, Value::ValueKind::kU8Array > U8Array
Typed UBJSON array for uint8_t.
Definition: json.h:177
xgboost::UBJReader::Load
Json Load() override
xgboost::JsonReader::GetNextChar
char GetNextChar()
Definition: json_io.h:49
xgboost::JsonReader::ParseBoolean
virtual Json ParseBoolean()
XGBOOST_EXPECT
#define XGBOOST_EXPECT(cond, ret)
Definition: base.h:75
xgboost::StringView::size
constexpr size_t size() const
Definition: string_view.h:38
xgboost::JsonReader::GetConsecutiveChar
char GetConsecutiveChar(char expected_char)
Definition: json_io.h:74
xgboost::JsonTypedArray
Typed array for Universal Binary JSON.
Definition: json.h:146
xgboost::JsonReader::~JsonReader
virtual ~JsonReader()=default
xgboost::JsonReader::ParseNull
virtual Json ParseNull()
xgboost::UBJReader
Reader for UBJSON https://ubjson.org/.
Definition: json_io.h:205
xgboost::StringView
Definition: string_view.h:14
xgboost::JsonReader::Expect
void Expect(char c, char got)
Definition: json_io.h:83
xgboost::BuiltinBSwap
T BuiltinBSwap(T v)
Definition: json_io.h:178
xgboost::JsonReader::kMaxNumLength
constexpr static size_t kMaxNumLength
Definition: json_io.h:25
base.h
defines configuration macros of xgboost.
xgboost::JsonReader::SourceLocation
Definition: json_io.h:28
xgboost::JsonReader::ParseObject
virtual Json ParseObject()
xgboost::JsonReader
Definition: json_io.h:23
xgboost::UBJWriter::Save
void Save(Json json) override
xgboost::JsonReader::ParseString
virtual Json ParseString()
xgboost::JsonReader::raw_str_
StringView raw_str_
Definition: json_io.h:44
xgboost::JsonReader::Load
virtual Json Load()
xgboost::JsonReader::GetNextNonSpaceChar
char GetNextNonSpaceChar()
Definition: json_io.h:67
xgboost::JsonNull
Definition: json.h:291
xgboost::I64Array
JsonTypedArray< int64_t, Value::ValueKind::kI64Array > I64Array
Typed UBJSON array for int64_t.
Definition: json.h:185
xgboost::JsonReader::ParseArray
virtual Json ParseArray()
xgboost::JsonReader::SkipSpaces
void SkipSpaces()
xgboost::JsonReader::SourceLocation::SourceLocation
SourceLocation()=default
xgboost::JsonReader::cursor_
struct xgboost::JsonReader::SourceLocation cursor_
xgboost::I32Array
JsonTypedArray< int32_t, Value::ValueKind::kI32Array > I32Array
Typed UBJSON array for int32_t.
Definition: json.h:181
xgboost::ToBigEndian
T ToBigEndian(T v)
Definition: json_io.h:185
xgboost::JsonWriter::stream_
std::vector< char > * stream_
Definition: json_io.h:136
xgboost::JsonReader::SourceLocation::Pos
size_t Pos() const
Definition: json_io.h:34
xgboost::JsonReader::SourceLocation::Forward
void Forward()
Definition: json_io.h:36
xgboost::F32Array
JsonTypedArray< float, Value::ValueKind::kNumberArray > F32Array
Typed UBJSON array for 32-bit floating point.
Definition: json.h:173
xgboost::JsonArray::GetArray
std::vector< Json > const & GetArray() &&
Definition: json.h:128
xgboost::JsonInteger
Definition: json.h:248
xgboost::JsonWriter::~JsonWriter
virtual ~JsonWriter()=default
json.h
xgboost::JsonWriter::JsonWriter
JsonWriter(std::vector< char > *stream)
Definition: json_io.h:139
xgboost::JsonReader::SourceLocation::Forward
void Forward(uint32_t n)
Definition: json_io.h:39
xgboost::JsonObject
Definition: json.h:187
xgboost::JsonArray
Definition: json.h:110
xgboost::JsonString
Definition: json.h:84
xgboost::StringView::c_str
CharT const * c_str() const
Definition: string_view.h:44
xgboost::JsonWriter
Definition: json_io.h:115
xgboost::JsonReader::ParseNumber
virtual Json ParseNumber()
xgboost::UBJWriter
Writer for UBJSON https://ubjson.org/.
Definition: json_io.h:247
xgboost::JsonReader::JsonReader
JsonReader(StringView str)
Definition: json_io.h:107
xgboost::Json
Data structure representing JSON format.
Definition: json.h:352
xgboost::JsonReader::Error
void Error(std::string msg) const
xgboost::JsonReader::PeekNextChar
char PeekNextChar()
Definition: json_io.h:58
xgboost
namespace of xgboost
Definition: base.h:110