xgboost
data.h
Go to the documentation of this file.
1 
7 #ifndef XGBOOST_DATA_H_
8 #define XGBOOST_DATA_H_
9 
10 #include <dmlc/base.h>
11 #include <dmlc/data.h>
12 #include <rabit/rabit.h>
13 #include <xgboost/base.h>
14 #include <xgboost/span.h>
16 
17 #include <memory>
18 #include <numeric>
19 #include <algorithm>
20 #include <string>
21 #include <utility>
22 #include <vector>
23 
24 namespace xgboost {
25 // forward declare dmatrix.
26 class DMatrix;
27 
29 enum class DataType : uint8_t {
30  kFloat32 = 1,
31  kDouble = 2,
32  kUInt32 = 3,
33  kUInt64 = 4
34 };
35 
39 class MetaInfo {
40  public:
42  static constexpr uint64_t kNumField = 7;
43 
45  uint64_t num_row_{0};
47  uint64_t num_col_{0};
49  uint64_t num_nonzero_{0};
56  std::vector<bst_group_t> group_ptr_;
65 
67  MetaInfo() = default;
68  MetaInfo& operator=(MetaInfo const& that) {
69  this->num_row_ = that.num_row_;
70  this->num_col_ = that.num_col_;
71  this->num_nonzero_ = that.num_nonzero_;
72 
73  this->labels_.Resize(that.labels_.Size());
74  this->labels_.Copy(that.labels_);
75 
76  this->group_ptr_ = that.group_ptr_;
77 
78  this->weights_.Resize(that.weights_.Size());
79  this->weights_.Copy(that.weights_);
80  this->base_margin_.Resize(that.base_margin_.Size());
81  this->base_margin_.Copy(that.base_margin_);
82  return *this;
83  }
89  inline bst_float GetWeight(size_t i) const {
90  return weights_.Size() != 0 ? weights_.HostVector()[i] : 1.0f;
91  }
93  inline const std::vector<size_t>& LabelAbsSort() const {
94  if (label_order_cache_.size() == labels_.Size()) {
95  return label_order_cache_;
96  }
97  label_order_cache_.resize(labels_.Size());
98  std::iota(label_order_cache_.begin(), label_order_cache_.end(), 0);
99  const auto& l = labels_.HostVector();
100  XGBOOST_PARALLEL_SORT(label_order_cache_.begin(), label_order_cache_.end(),
101  [&l](size_t i1, size_t i2) {return std::abs(l[i1]) < std::abs(l[i2]);});
102 
103  return label_order_cache_;
104  }
106  void Clear();
111  void LoadBinary(dmlc::Stream* fi);
116  void SaveBinary(dmlc::Stream* fo) const;
124  void SetInfo(const char* key, const void* dptr, DataType dtype, size_t num);
134  void SetInfo(const char* key, std::string const& interface_str);
135 
136  private:
138  mutable std::vector<size_t> label_order_cache_;
139 };
140 
142 struct Entry {
148  Entry() = default;
154  XGBOOST_DEVICE Entry(bst_feature_t index, bst_float fvalue) : index(index), fvalue(fvalue) {}
156  inline static bool CmpValue(const Entry& a, const Entry& b) {
157  return a.fvalue < b.fvalue;
158  }
159  inline bool operator==(const Entry& other) const {
160  return (this->index == other.index && this->fvalue == other.fvalue);
161  }
162 };
163 
167 struct BatchParam {
169  int gpu_id;
171  int max_bin;
176 
177  inline bool operator!=(const BatchParam& other) const {
178  return gpu_id != other.gpu_id ||
179  max_bin != other.max_bin ||
180  gpu_batch_nrows != other.gpu_batch_nrows ||
181  gpu_page_size != other.gpu_page_size;
182  }
183 };
184 
188 class SparsePage {
189  public:
190  // Offset for each row.
194 
195  size_t base_rowid{};
196 
199 
201  inline Inst operator[](size_t i) const {
202  const auto& data_vec = data.HostVector();
203  const auto& offset_vec = offset.HostVector();
204  size_t size;
205  // in distributed mode, some partitions may not get any instance for a feature. Therefore
206  // we should set the size as zero
207  if (rabit::IsDistributed() && i + 1 >= offset_vec.size()) {
208  size = 0;
209  } else {
210  size = offset_vec[i + 1] - offset_vec[i];
211  }
212  return {data_vec.data() + offset_vec[i],
213  static_cast<Inst::index_type>(size)};
214  }
215 
218  this->Clear();
219  }
220 
222  inline size_t Size() const {
223  return offset.Size() == 0 ? 0 : offset.Size() - 1;
224  }
225 
227  inline size_t MemCostBytes() const {
228  return offset.Size() * sizeof(size_t) + data.Size() * sizeof(Entry);
229  }
230 
232  inline void Clear() {
233  base_rowid = 0;
234  auto& offset_vec = offset.HostVector();
235  offset_vec.clear();
236  offset_vec.push_back(0);
237  data.HostVector().clear();
238  }
239 
241  inline void SetBaseRowId(size_t row_id) {
242  base_rowid = row_id;
243  }
244 
245  SparsePage GetTranspose(int num_columns) const;
246 
247  void SortRows() {
248  auto ncol = static_cast<bst_omp_uint>(this->Size());
249 #pragma omp parallel for default(none) shared(ncol) schedule(dynamic, 1)
250  for (bst_omp_uint i = 0; i < ncol; ++i) {
251  if (this->offset.HostVector()[i] < this->offset.HostVector()[i + 1]) {
252  std::sort(
253  this->data.HostVector().begin() + this->offset.HostVector()[i],
254  this->data.HostVector().begin() + this->offset.HostVector()[i + 1],
256  }
257  }
258  }
259 
264  void Push(const dmlc::RowBlock<uint32_t>& batch);
265 
276  template <typename AdapterBatchT>
277  uint64_t Push(const AdapterBatchT& batch, float missing, int nthread);
278 
283  void Push(const SparsePage &batch);
288  void PushCSC(const SparsePage& batch);
289 };
290 
291 class CSCPage: public SparsePage {
292  public:
294  explicit CSCPage(SparsePage page) : SparsePage(std::move(page)) {}
295 };
296 
297 class SortedCSCPage : public SparsePage {
298  public:
300  explicit SortedCSCPage(SparsePage page) : SparsePage(std::move(page)) {}
301 };
302 
303 class EllpackPageImpl;
310 class EllpackPage {
311  public:
318  EllpackPage();
319 
326  explicit EllpackPage(DMatrix* dmat, const BatchParam& param);
327 
329  ~EllpackPage();
330 
332  size_t Size() const;
333 
335  void SetBaseRowId(size_t row_id);
336 
337  const EllpackPageImpl* Impl() const { return impl_.get(); }
338  EllpackPageImpl* Impl() { return impl_.get(); }
339 
340  private:
341  std::unique_ptr<EllpackPageImpl> impl_;
342 };
343 
344 template<typename T>
346  public:
347  virtual ~BatchIteratorImpl() = default;
348  virtual T& operator*() = 0;
349  virtual const T& operator*() const = 0;
350  virtual void operator++() = 0;
351  virtual bool AtEnd() const = 0;
352 };
353 
354 template<typename T>
356  public:
357  using iterator_category = std::forward_iterator_tag;
358  explicit BatchIterator(BatchIteratorImpl<T>* impl) { impl_.reset(impl); }
359 
360  void operator++() {
361  CHECK(impl_ != nullptr);
362  ++(*impl_);
363  }
364 
365  T& operator*() {
366  CHECK(impl_ != nullptr);
367  return *(*impl_);
368  }
369 
370  const T& operator*() const {
371  CHECK(impl_ != nullptr);
372  return *(*impl_);
373  }
374 
375  bool operator!=(const BatchIterator& rhs) const {
376  CHECK(impl_ != nullptr);
377  return !impl_->AtEnd();
378  }
379 
380  bool AtEnd() const {
381  CHECK(impl_ != nullptr);
382  return impl_->AtEnd();
383  }
384 
385  private:
386  std::shared_ptr<BatchIteratorImpl<T>> impl_;
387 };
388 
389 template<typename T>
390 class BatchSet {
391  public:
392  explicit BatchSet(BatchIterator<T> begin_iter) : begin_iter_(begin_iter) {}
393  BatchIterator<T> begin() { return begin_iter_; }
394  BatchIterator<T> end() { return BatchIterator<T>(nullptr); }
395 
396  private:
397  BatchIterator<T> begin_iter_;
398 };
399 
407 template<typename T>
408 class DataSource : public dmlc::DataIter<T> {
409  public:
415 };
416 
428 class DMatrix {
429  public:
431  DMatrix() = default;
433  virtual MetaInfo& Info() = 0;
435  virtual const MetaInfo& Info() const = 0;
439  template<typename T>
440  BatchSet<T> GetBatches(const BatchParam& param = {});
441  // the following are column meta data, should be able to answer them fast.
443  virtual bool SingleColBlock() const = 0;
445  virtual float GetColDensity(size_t cidx) = 0;
447  virtual ~DMatrix() = default;
455  virtual void SaveToLocalFile(const std::string& fname);
456 
458  bool IsDense() const {
459  return Info().num_nonzero_ == Info().num_row_ * Info().num_col_;
460  }
461 
472  static DMatrix* Load(const std::string& uri,
473  bool silent,
474  bool load_row_split,
475  const std::string& file_format = "auto",
476  size_t page_size = kPageSize);
477 
485  static DMatrix* Create(std::unique_ptr<DataSource<SparsePage>>&& source,
486  const std::string& cache_prefix = "");
487 
500  template <typename AdapterT>
501  static DMatrix* Create(AdapterT* adapter, float missing, int nthread,
502  const std::string& cache_prefix = "",
503  size_t page_size = kPageSize);
504 
505 
507  static const size_t kPageSize = 32UL << 20UL;
508 
509  protected:
510  virtual BatchSet<SparsePage> GetRowBatches() = 0;
511  virtual BatchSet<CSCPage> GetColumnBatches() = 0;
512  virtual BatchSet<SortedCSCPage> GetSortedColumnBatches() = 0;
513  virtual BatchSet<EllpackPage> GetEllpackBatches(const BatchParam& param) = 0;
514 };
515 
516 template<>
518  return GetRowBatches();
519 }
520 
521 template<>
523  return GetColumnBatches();
524 }
525 
526 template<>
528  return GetSortedColumnBatches();
529 }
530 
531 template<>
533  return GetEllpackBatches(param);
534 }
535 } // namespace xgboost
536 
537 namespace dmlc {
538 DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true);
539 }
540 #endif // XGBOOST_DATA_H_
std::vector< bst_group_t > group_ptr_
the index of begin and end of a group needed when the learning task is ranking.
Definition: data.h:56
Definition: data.h:355
void operator++()
Definition: data.h:360
uint64_t num_col_
number of columns in the data
Definition: data.h:47
std::forward_iterator_tag iterator_category
Definition: data.h:357
float bst_float
float type, used for storing statistics
Definition: base.h:111
bool IsDense() const
Whether the matrix is dense.
Definition: data.h:458
void Copy(const HostDeviceVector< T > &other)
BatchIterator(BatchIteratorImpl< T > *impl)
Definition: data.h:358
#define XGBOOST_PARALLEL_SORT(X, Y, Z)
Definition: base.h:68
A page stored in ELLPACK format.
Definition: data.h:310
const T & operator*() const
Definition: data.h:370
T & operator*()
Definition: data.h:365
SortedCSCPage()
Definition: data.h:299
std::size_t index_type
Definition: span.h:394
bool operator!=(const BatchParam &other) const
Definition: data.h:177
Meta information about dataset, always sit in memory.
Definition: data.h:39
bool AtEnd() const
Definition: data.h:380
void SetBaseRowId(size_t row_id)
Set the base row id for this page.
Definition: data.h:241
void SortRows()
Definition: data.h:247
size_t gpu_page_size
Page size for external memory mode.
Definition: data.h:175
BatchSet(BatchIterator< T > begin_iter)
Definition: data.h:392
uint32_t bst_feature_t
Type for data column (feature) index.
Definition: base.h:114
dmlc::omp_uint bst_omp_uint
define unsigned int for openmp loop
Definition: base.h:246
Internal data structured used by XGBoost during training. There are two ways to create a customized D...
Definition: data.h:428
In-memory storage unit of sparse batch, stored in CSR format.
Definition: data.h:188
A device-and-host vector abstraction layer.
Definition: data.h:297
Parameters for constructing batches.
Definition: data.h:167
MetaInfo & operator=(MetaInfo const &that)
Definition: data.h:68
span class implementation, based on ISO++20 span<T>. The interface should be the same.
Definition: span.h:115
BatchSet< T > GetBatches(const BatchParam &param={})
Gets batches. Use range based for loop over BatchSet to access individual batches.
int gpu_id
The GPU device to use.
Definition: data.h:169
Definition: data.h:390
HostDeviceVector< bst_row_t > offset
Definition: data.h:191
XGBOOST_DEVICE Entry(bst_feature_t index, bst_float fvalue)
constructor with index and value
Definition: data.h:154
SparsePage()
constructor
Definition: data.h:217
Definition: data.h:291
BatchIterator< T > begin()
Definition: data.h:393
const EllpackPageImpl * Impl() const
Definition: data.h:337
Definition: data.h:345
SortedCSCPage(SparsePage page)
Definition: data.h:300
bst_float GetWeight(size_t i) const
Get weight of each instances.
Definition: data.h:89
Definition: data.h:537
CSCPage()
Definition: data.h:293
std::vector< T > & HostVector()
bool operator==(const Entry &other) const
Definition: data.h:159
HostDeviceVector< bst_float > labels_
label of each instance
Definition: data.h:51
Inst operator[](size_t i) const
get i-th row from the batch
Definition: data.h:201
uint64_t num_row_
number of rows in the data
Definition: data.h:45
This is data structure that user can pass to DMatrix::Create to create a DMatrix for training...
Definition: data.h:408
bool operator!=(const BatchIterator &rhs) const
Definition: data.h:375
#define XGBOOST_DEVICE
Tag function as usable by device.
Definition: base.h:84
HostDeviceVector< bst_float > base_margin_
initialized margins, if specified, xgboost will start from this init margin can be used to specify in...
Definition: data.h:64
size_t MemCostBytes() const
Definition: data.h:227
namespace of xgboost
Definition: base.h:102
CSCPage(SparsePage page)
Definition: data.h:294
defines configuration macros of xgboost.
size_t Size() const
Definition: data.h:222
DataType
data type accepted by xgboost interface
Definition: data.h:29
HostDeviceVector< Entry > data
the data of the segments
Definition: data.h:193
uint64_t num_nonzero_
number of nonzero entries in the data
Definition: data.h:49
Element from a sparse vector.
Definition: data.h:142
DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true)
BatchIterator< T > end()
Definition: data.h:394
bst_feature_t index
feature index
Definition: data.h:144
const std::vector< size_t > & LabelAbsSort() const
get sorted indexes (argsort) of labels by absolute value (used by cox loss)
Definition: data.h:93
HostDeviceVector< bst_float > weights_
weights of each instance, optional
Definition: data.h:58
bst_float fvalue
feature value
Definition: data.h:146
static bool CmpValue(const Entry &a, const Entry &b)
reversely compare feature values
Definition: data.h:156
void Resize(size_t new_size, T v=T())
void Clear()
clear the page
Definition: data.h:232
int gpu_batch_nrows
Number of rows in a GPU batch, used for finding quantiles on GPU.
Definition: data.h:173
int max_bin
Maximum number of bins per feature for histograms.
Definition: data.h:171
MetaInfo info
Meta information about the dataset The subclass need to be able to load this correctly from data...
Definition: data.h:414
EllpackPageImpl * Impl()
Definition: data.h:338