xgboost
json_io.h
Go to the documentation of this file.
1 
4 #ifndef XGBOOST_JSON_IO_H_
5 #define XGBOOST_JSON_IO_H_
6 #include <dmlc/endian.h>
7 #include <xgboost/base.h>
8 #include <xgboost/json.h>
9 
10 #include <cinttypes>
11 #include <limits>
12 #include <map>
13 #include <memory>
14 #include <sstream>
15 #include <string>
16 #include <utility>
17 #include <vector>
18 
19 namespace xgboost {
20 /*
21  * \brief A json reader, currently error checking and utf-8 is not fully supported.
22  */
23 class JsonReader {
24  protected:
25  size_t constexpr static kMaxNumLength =
26  std::numeric_limits<double>::max_digits10 + 1;
27 
28  struct SourceLocation {
29  private:
30  size_t pos_ { 0 }; // current position in raw_str_
31 
32  public:
33  SourceLocation() = default;
34  size_t Pos() const { return pos_; }
35 
36  void Forward() {
37  pos_++;
38  }
39  void Forward(uint32_t n) {
40  pos_ += n;
41  }
42  } cursor_;
43 
45 
46  protected:
47  void SkipSpaces();
48 
49  char GetNextChar() {
50  if (XGBOOST_EXPECT((cursor_.Pos() == raw_str_.size()), false)) {
51  return -1;
52  }
53  char ch = raw_str_[cursor_.Pos()];
54  cursor_.Forward();
55  return ch;
56  }
57 
58  char PeekNextChar() {
59  if (cursor_.Pos() == raw_str_.size()) {
60  return -1;
61  }
62  char ch = raw_str_[cursor_.Pos()];
63  return ch;
64  }
65 
66  /* \brief Skip spaces and consume next character. */
68  SkipSpaces();
69  return GetNextChar();
70  }
71  /* \brief Consume next character without first skipping empty space, throw when the next
72  * character is not the expected one.
73  */
74  char GetConsecutiveChar(char expected_char) {
75  char result = GetNextChar();
76  if (XGBOOST_EXPECT(result != expected_char, false)) { Expect(expected_char, result); }
77  return result;
78  }
79 
80  void Error(std::string msg) const;
81 
82  // Report expected character
83  void Expect(char c, char got) {
84  std::string msg = "Expecting: \"";
85  msg += c;
86  msg += "\", got: \"";
87  if (got == EOF) {
88  msg += "EOF\"";
89  } else if (got == 0) {
90  msg += "\\0\"";
91  } else {
92  msg += (got <= 127 ? std::string{got} : std::to_string(got)) + " \""; // NOLINT
93  }
94  Error(msg);
95  }
96 
97  virtual Json ParseString();
98  virtual Json ParseObject();
99  virtual Json ParseArray();
100  virtual Json ParseNumber();
101  virtual Json ParseBoolean();
102  virtual Json ParseNull();
103 
104  Json Parse();
105 
106  public:
107  explicit JsonReader(StringView str) :
108  raw_str_{str} {}
109 
110  virtual ~JsonReader() = default;
111 
112  virtual Json Load();
113 };
114 
115 class JsonWriter {
116  template <typename T, std::enable_if_t<!std::is_same<Json, T>::value>* = nullptr>
117  void Save(T const& v) {
118  this->Save(Json{v});
119  }
120  template <typename Array, typename Fn>
121  void WriteArray(Array const* arr, Fn&& fn) {
122  stream_->emplace_back('[');
123  auto const& vec = arr->GetArray();
124  size_t size = vec.size();
125  for (size_t i = 0; i < size; ++i) {
126  auto const& value = vec[i];
127  this->Save(fn(value));
128  if (i != size - 1) {
129  stream_->emplace_back(',');
130  }
131  }
132  stream_->emplace_back(']');
133  }
134 
135  protected:
136  std::vector<char>* stream_;
137 
138  public:
139  explicit JsonWriter(std::vector<char>* stream) : stream_{stream} {}
140 
141  virtual ~JsonWriter() = default;
142 
143  virtual void Save(Json json);
144 
145  virtual void Visit(JsonArray const* arr);
146  virtual void Visit(F32Array const* arr);
147  virtual void Visit(U8Array const* arr);
148  virtual void Visit(I32Array const* arr);
149  virtual void Visit(I64Array const* arr);
150  virtual void Visit(JsonObject const* obj);
151  virtual void Visit(JsonNumber const* num);
152  virtual void Visit(JsonInteger const* num);
153  virtual void Visit(JsonNull const* null);
154  virtual void Visit(JsonString const* str);
155  virtual void Visit(JsonBoolean const* boolean);
156 };
157 
158 #if defined(__GLIBC__)
159 template <typename T>
160 T BuiltinBSwap(T v);
161 
162 template <>
163 inline uint16_t BuiltinBSwap(uint16_t v) {
164  return __builtin_bswap16(v);
165 }
166 
167 template <>
168 inline uint32_t BuiltinBSwap(uint32_t v) {
169  return __builtin_bswap32(v);
170 }
171 
172 template <>
173 inline uint64_t BuiltinBSwap(uint64_t v) {
174  return __builtin_bswap64(v);
175 }
176 #else
177 template <typename T>
178 T BuiltinBSwap(T v) {
179  dmlc::ByteSwap(&v, sizeof(v), 1);
180  return v;
181 }
182 #endif // defined(__GLIBC__)
183 
184 template <typename T, std::enable_if_t<sizeof(T) == 1>* = nullptr>
185 inline T ToBigEndian(T v) {
186  return v;
187 }
188 
189 template <typename T, std::enable_if_t<sizeof(T) != 1>* = nullptr>
190 inline T ToBigEndian(T v) {
191  static_assert(std::is_pod<T>::value, "Only pod is supported.");
192 #if DMLC_LITTLE_ENDIAN
193  auto constexpr kS = sizeof(T);
194  std::conditional_t<kS == 2, uint16_t, std::conditional_t<kS == 4, uint32_t, uint64_t>> u;
195  std::memcpy(&u, &v, sizeof(u));
196  u = BuiltinBSwap(u);
197  std::memcpy(&v, &u, sizeof(u));
198 #endif // DMLC_LITTLE_ENDIAN
199  return v;
200 }
201 
205 class UBJReader : public JsonReader {
206  Json Parse();
207 
208  template <typename T>
209  T ReadStream() {
210  auto ptr = this->raw_str_.c_str() + cursor_.Pos();
211  T v{0};
212  std::memcpy(&v, ptr, sizeof(v));
213  cursor_.Forward(sizeof(v));
214  return v;
215  }
216 
217  template <typename T>
218  T ReadPrimitive() {
219  auto v = ReadStream<T>();
220  v = ToBigEndian(v);
221  return v;
222  }
223 
224  template <typename TypedArray>
225  auto ParseTypedArray(int64_t n) {
226  TypedArray results{static_cast<size_t>(n)};
227  for (int64_t i = 0; i < n; ++i) {
228  auto v = this->ReadPrimitive<typename TypedArray::Type>();
229  results.Set(i, v);
230  }
231  return Json{std::move(results)};
232  }
233 
234  std::string DecodeStr();
235 
236  Json ParseArray() override;
237  Json ParseObject() override;
238 
239  public:
241  Json Load() override;
242 };
243 
247 class UBJWriter : public JsonWriter {
248  void Visit(JsonArray const* arr) override;
249  void Visit(F32Array const* arr) override;
250  void Visit(U8Array const* arr) override;
251  void Visit(I32Array const* arr) override;
252  void Visit(I64Array const* arr) override;
253  void Visit(JsonObject const* obj) override;
254  void Visit(JsonNumber const* num) override;
255  void Visit(JsonInteger const* num) override;
256  void Visit(JsonNull const* null) override;
257  void Visit(JsonString const* str) override;
258  void Visit(JsonBoolean const* boolean) override;
259 
260  public:
262  void Save(Json json) override;
263 };
264 } // namespace xgboost
265 
266 #endif // XGBOOST_JSON_IO_H_
xgboost::JsonBoolean
Describes both true and false.
Definition: json.h:307
xgboost::JsonWriter::Visit
virtual void Visit(JsonArray const *arr)
xgboost::JsonNumber
Definition: json.h:214
xgboost::JsonReader::Parse
Json Parse()
xgboost::U8Array
JsonTypedArray< uint8_t, Value::ValueKind::kU8Array > U8Array
Typed UBJSON array for uint8_t.
Definition: json.h:177
xgboost::UBJReader::Load
Json Load() override
xgboost::JsonReader::GetNextChar
char GetNextChar()
Definition: json_io.h:49
xgboost::JsonReader::ParseBoolean
virtual Json ParseBoolean()
XGBOOST_EXPECT
#define XGBOOST_EXPECT(cond, ret)
Definition: base.h:75
xgboost::StringView::size
constexpr size_t size() const
Definition: string_view.h:38
xgboost::JsonReader::GetConsecutiveChar
char GetConsecutiveChar(char expected_char)
Definition: json_io.h:74
xgboost::JsonTypedArray
Typed array for Universal Binary JSON.
Definition: json.h:146
xgboost::JsonReader::~JsonReader
virtual ~JsonReader()=default
xgboost::JsonReader::ParseNull
virtual Json ParseNull()
xgboost::UBJReader
Reader for UBJSON https://ubjson.org/.
Definition: json_io.h:205
xgboost::StringView
Definition: string_view.h:14
xgboost::JsonReader::Expect
void Expect(char c, char got)
Definition: json_io.h:83
xgboost::BuiltinBSwap
T BuiltinBSwap(T v)
Definition: json_io.h:178
xgboost::JsonReader::kMaxNumLength
constexpr static size_t kMaxNumLength
Definition: json_io.h:25
base.h
defines configuration macros of xgboost.
xgboost::JsonReader::SourceLocation
Definition: json_io.h:28
xgboost::JsonReader::ParseObject
virtual Json ParseObject()
xgboost::JsonReader
Definition: json_io.h:23
xgboost::UBJWriter::Save
void Save(Json json) override
xgboost::JsonReader::ParseString
virtual Json ParseString()
xgboost::JsonReader::raw_str_
StringView raw_str_
Definition: json_io.h:44
xgboost::JsonReader::Load
virtual Json Load()
xgboost::JsonReader::GetNextNonSpaceChar
char GetNextNonSpaceChar()
Definition: json_io.h:67
xgboost::JsonNull
Definition: json.h:291
xgboost::I64Array
JsonTypedArray< int64_t, Value::ValueKind::kI64Array > I64Array
Typed UBJSON array for int64_t.
Definition: json.h:185
xgboost::JsonReader::ParseArray
virtual Json ParseArray()
xgboost::JsonReader::SkipSpaces
void SkipSpaces()
xgboost::JsonReader::SourceLocation::SourceLocation
SourceLocation()=default
xgboost::JsonReader::cursor_
struct xgboost::JsonReader::SourceLocation cursor_
xgboost::I32Array
JsonTypedArray< int32_t, Value::ValueKind::kI32Array > I32Array
Typed UBJSON array for int32_t.
Definition: json.h:181
xgboost::ToBigEndian
T ToBigEndian(T v)
Definition: json_io.h:185
xgboost::JsonWriter::stream_
std::vector< char > * stream_
Definition: json_io.h:136
xgboost::JsonReader::SourceLocation::Pos
size_t Pos() const
Definition: json_io.h:34
xgboost::JsonReader::SourceLocation::Forward
void Forward()
Definition: json_io.h:36
xgboost::F32Array
JsonTypedArray< float, Value::ValueKind::kNumberArray > F32Array
Typed UBJSON array for 32-bit floating point.
Definition: json.h:173
xgboost::JsonArray::GetArray
std::vector< Json > const & GetArray() &&
Definition: json.h:128
xgboost::JsonInteger
Definition: json.h:248
xgboost::JsonWriter::~JsonWriter
virtual ~JsonWriter()=default
json.h
xgboost::JsonWriter::JsonWriter
JsonWriter(std::vector< char > *stream)
Definition: json_io.h:139
xgboost::JsonReader::SourceLocation::Forward
void Forward(uint32_t n)
Definition: json_io.h:39
xgboost::JsonObject
Definition: json.h:187
xgboost::JsonArray
Definition: json.h:110
xgboost::JsonString
Definition: json.h:84
xgboost::StringView::c_str
CharT const * c_str() const
Definition: string_view.h:44
xgboost::JsonWriter
Definition: json_io.h:115
xgboost::JsonReader::ParseNumber
virtual Json ParseNumber()
xgboost::UBJWriter
Writer for UBJSON https://ubjson.org/.
Definition: json_io.h:247
xgboost::JsonReader::JsonReader
JsonReader(StringView str)
Definition: json_io.h:107
xgboost::Json
Data structure representing JSON format.
Definition: json.h:352
xgboost::JsonReader::Error
void Error(std::string msg) const
xgboost::JsonReader::PeekNextChar
char PeekNextChar()
Definition: json_io.h:58
xgboost
namespace of xgboost
Definition: base.h:110