xgboost
data.h
Go to the documentation of this file.
1 
7 #ifndef XGBOOST_DATA_H_
8 #define XGBOOST_DATA_H_
9 
10 #include <dmlc/base.h>
11 #include <dmlc/data.h>
12 #include <dmlc/serializer.h>
13 #include <xgboost/base.h>
15 #include <xgboost/linalg.h>
16 #include <xgboost/span.h>
17 #include <xgboost/string_view.h>
18 
19 #include <algorithm>
20 #include <cstdint> // for int32_t, uint8_t
21 #include <limits>
22 #include <memory>
23 #include <string>
24 #include <utility>
25 #include <vector>
26 
27 namespace xgboost {
28 // forward declare dmatrix.
29 class DMatrix;
30 struct Context;
31 
33 enum class DataType : uint8_t {
34  kFloat32 = 1,
35  kDouble = 2,
36  kUInt32 = 3,
37  kUInt64 = 4,
38  kStr = 5
39 };
40 
41 enum class FeatureType : uint8_t { kNumerical = 0, kCategorical = 1 };
42 
43 enum class DataSplitMode : int { kRow = 0, kCol = 1 };
44 
48 class MetaInfo {
49  public:
51  static constexpr uint64_t kNumField = 12;
52 
54  bst_idx_t num_row_{0}; // NOLINT
56  uint64_t num_col_{0}; // NOLINT
58  uint64_t num_nonzero_{0}; // NOLINT
67  std::vector<bst_group_t> group_ptr_; // NOLINT
84 
88  std::vector<std::string> feature_type_names;
92  std::vector<std::string> feature_names;
93  /*
94  * \brief Type of each feature. Automatically set when feature_type_names is specifed.
95  */
97  /*
98  * \brief Weight of each feature, used to define the probability of each feature being
99  * selected when using column sampling.
100  */
102 
104  MetaInfo() = default;
105  MetaInfo(MetaInfo&& that) = default;
106  MetaInfo& operator=(MetaInfo&& that) = default;
107  MetaInfo& operator=(MetaInfo const& that) = delete;
108 
112  void Validate(DeviceOrd device) const;
122 
123  MetaInfo Copy() const;
127  bool IsDense() const { return num_col_ * num_row_ == num_nonzero_; }
133  inline bst_float GetWeight(size_t i) const {
134  return weights_.Size() != 0 ? weights_.HostVector()[i] : 1.0f;
135  }
137  const std::vector<size_t>& LabelAbsSort(Context const* ctx) const;
139  void Clear();
144  void LoadBinary(dmlc::Stream* fi);
149  void SaveBinary(dmlc::Stream* fo) const;
155  void SetInfo(Context const& ctx, StringView key, StringView interface_str);
156 
157  void GetInfo(char const* key, bst_ulong* out_len, DataType dtype,
158  const void** out_dptr) const;
159 
160  void SetFeatureInfo(const char *key, const char **info, const bst_ulong size);
161  void GetFeatureInfo(const char *field, std::vector<std::string>* out_str_vecs) const;
162 
174  void Extend(MetaInfo const& that, bool accumulate_rows, bool check_column);
182  void SynchronizeNumberOfColumns(Context const* ctx, DataSplitMode split_mode);
183 
185  [[nodiscard]] bool IsRowSplit() const { return data_split_mode == DataSplitMode::kRow; }
187  [[nodiscard]] bool IsColumnSplit() const { return data_split_mode == DataSplitMode::kCol; }
189  [[nodiscard]] bool IsRanking() const { return !group_ptr_.empty(); }
190 
195  [[nodiscard]] bool IsVerticalFederated() const;
196 
203  bool ShouldHaveLabels() const;
207  bool HasCategorical() const { return has_categorical_; }
208 
209  private:
210  void SetInfoFromHost(Context const* ctx, StringView key, Json arr);
211  void SetInfoFromCUDA(Context const* ctx, StringView key, Json arr);
212 
214  mutable std::vector<size_t> label_order_cache_;
215  bool has_categorical_{false};
216 };
217 
219 struct Entry {
225  Entry() = default;
233  inline static bool CmpValue(const Entry& a, const Entry& b) {
234  return a.fvalue < b.fvalue;
235  }
236  static bool CmpIndex(Entry const& a, Entry const& b) {
237  return a.index < b.index;
238  }
239  inline bool operator==(const Entry& other) const {
240  return (this->index == other.index && this->fvalue == other.fvalue);
241  }
242 };
243 
247 struct BatchParam {
260  bool regen{false};
264  bool forbid_regen{false};
268  double sparse_thresh{std::numeric_limits<double>::quiet_NaN()};
274  bool prefetch_copy{true};
278  std::int32_t n_prefetch_batches{3};
279 
283  BatchParam() = default;
296  : max_bin{max_bin}, hess{hessian}, regen{regenerate} {}
297 
298  [[nodiscard]] bool ParamNotEqual(BatchParam const& other) const {
299  // Check non-floating parameters.
300  bool cond = max_bin != other.max_bin;
301  // Check sparse thresh.
302  bool l_nan = std::isnan(sparse_thresh);
303  bool r_nan = std::isnan(other.sparse_thresh);
304  bool st_chg = (l_nan != r_nan) || (!l_nan && !r_nan && (sparse_thresh != other.sparse_thresh));
305  cond |= st_chg;
306 
307  return cond;
308  }
309  [[nodiscard]] bool Initialized() const { return max_bin != 0; }
313  [[nodiscard]] BatchParam MakeCache() const {
314  auto p = *this;
315  // These parameters have nothing to do with how the gradient index was generated in the
316  // first place.
317  p.regen = false;
318  p.forbid_regen = false;
319  return p;
320  }
321 };
322 
325 
328 
329  [[nodiscard]] Inst operator[](std::size_t i) const {
330  auto size = *(offset.data() + i + 1) - *(offset.data() + i);
331  return {data.data() + *(offset.data() + i), static_cast<Inst::index_type>(size)};
332  }
333 
334  [[nodiscard]] size_t Size() const { return offset.size() == 0 ? 0 : offset.size() - 1; }
335 };
336 
340 class SparsePage {
341  public:
342  // Offset for each row.
346 
347  size_t base_rowid {0};
348 
351 
352  [[nodiscard]] HostSparsePageView GetView() const {
353  return {offset.ConstHostSpan(), data.ConstHostSpan()};
354  }
355 
358  this->Clear();
359  }
360 
361  SparsePage(SparsePage const& that) = delete;
362  SparsePage(SparsePage&& that) = default;
363  SparsePage& operator=(SparsePage const& that) = delete;
364  SparsePage& operator=(SparsePage&& that) = default;
365  virtual ~SparsePage() = default;
366 
368  [[nodiscard]] size_t Size() const {
369  return offset.Size() == 0 ? 0 : offset.Size() - 1;
370  }
371 
373  [[nodiscard]] size_t MemCostBytes() const {
374  return offset.Size() * sizeof(size_t) + data.Size() * sizeof(Entry);
375  }
376 
378  inline void Clear() {
379  base_rowid = 0;
380  auto& offset_vec = offset.HostVector();
381  offset_vec.clear();
382  offset_vec.push_back(0);
383  data.HostVector().clear();
384  }
385 
387  inline void SetBaseRowId(size_t row_id) {
388  base_rowid = row_id;
389  }
390 
391  [[nodiscard]] SparsePage GetTranspose(int num_columns, int32_t n_threads) const;
392 
396  void SortIndices(int32_t n_threads);
400  [[nodiscard]] bool IsIndicesSorted(int32_t n_threads) const;
404  void Reindex(uint64_t feature_offset, int32_t n_threads);
405 
406  void SortRows(int32_t n_threads);
407 
418  template <typename AdapterBatchT>
419  uint64_t Push(const AdapterBatchT& batch, float missing, int nthread);
420 
425  void Push(const SparsePage &batch);
430  void PushCSC(const SparsePage& batch);
431 };
432 
433 class CSCPage: public SparsePage {
434  public:
436  explicit CSCPage(SparsePage page) : SparsePage(std::move(page)) {}
437 };
438 
444  public:
445  std::shared_ptr<SparsePage const> page;
446  explicit ExtSparsePage(std::shared_ptr<SparsePage const> p) : page{std::move(p)} {}
447 };
448 
449 class SortedCSCPage : public SparsePage {
450  public:
452  explicit SortedCSCPage(SparsePage page) : SparsePage(std::move(page)) {}
453 };
454 
455 class EllpackPage;
456 class GHistIndexMatrix;
457 
458 template<typename T>
460  public:
461  using iterator_category = std::forward_iterator_tag; // NOLINT
462  virtual ~BatchIteratorImpl() = default;
463  virtual const T& operator*() const = 0;
465  [[nodiscard]] virtual bool AtEnd() const = 0;
466  virtual std::shared_ptr<T const> Page() const = 0;
467 };
468 
469 template<typename T>
471  public:
472  using iterator_category = std::forward_iterator_tag; // NOLINT
473  explicit BatchIterator(BatchIteratorImpl<T>* impl) { impl_.reset(impl); }
474  explicit BatchIterator(std::shared_ptr<BatchIteratorImpl<T>> impl) { impl_ = impl; }
475 
477  CHECK(impl_ != nullptr);
478  ++(*impl_);
479  return *this;
480  }
481 
482  const T& operator*() const {
483  CHECK(impl_ != nullptr);
484  return *(*impl_);
485  }
486 
487  [[nodiscard]] bool operator!=(const BatchIterator&) const { return !this->AtEnd(); }
488 
489  [[nodiscard]] bool AtEnd() const {
490  CHECK(impl_ != nullptr);
491  return impl_->AtEnd();
492  }
493 
494  [[nodiscard]] std::shared_ptr<T const> Page() const {
495  return impl_->Page();
496  }
497 
498  private:
499  std::shared_ptr<BatchIteratorImpl<T>> impl_;
500 };
501 
502 template<typename T>
503 class BatchSet {
504  public:
505  explicit BatchSet(BatchIterator<T> begin_iter) : begin_iter_(std::move(begin_iter)) {}
506  BatchIterator<T> begin() { return begin_iter_; } // NOLINT
507  BatchIterator<T> end() { return BatchIterator<T>(nullptr); } // NOLINT
508 
509  private:
510  BatchIterator<T> begin_iter_;
511 };
512 
513 struct XGBAPIThreadLocalEntry;
514 
515 // Configuration for external memoroy DMatrix.
516 struct ExtMemConfig {
517  // Cache prefix, not used if the cache is in the host memory. (on_host is true)
518  std::string cache;
519  // Whether the ellpack page is stored in the host memory.
520  bool on_host{true};
521  // Minimum number of of bytes for each ellpack page in cache. Only used for in-host
522  // ExtMemQdm.
523  std::int64_t min_cache_page_bytes{0};
524  // Missing value.
525  float missing{std::numeric_limits<float>::quiet_NaN()};
526  // Maximum number of pages cached in device.
527  std::int64_t max_num_device_pages{0};
528  // The number of CPU threads.
529  std::int32_t n_threads{0};
530 
531  ExtMemConfig() = default;
532  ExtMemConfig(std::string cache, bool on_host, std::int64_t min_cache, float missing,
533  std::int64_t max_num_d, std::int32_t n_threads)
534  : cache{std::move(cache)},
535  on_host{on_host},
536  min_cache_page_bytes{min_cache},
537  missing{missing},
538  max_num_device_pages{max_num_d},
539  n_threads{n_threads} {}
540 };
541 
549 class DMatrix {
550  public:
552  DMatrix() = default;
554  [[nodiscard]] virtual MetaInfo& Info() = 0;
555  virtual void SetInfo(const char* key, std::string const& interface_str) {
556  auto const& ctx = *this->Ctx();
557  this->Info().SetInfo(ctx, key, StringView{interface_str});
558  }
560  [[nodiscard]] virtual const MetaInfo& Info() const = 0;
561 
563  [[nodiscard]] XGBAPIThreadLocalEntry& GetThreadLocal() const;
568  [[nodiscard]] virtual Context const* Ctx() const = 0;
569 
573  template <typename T>
575  template <typename T>
577  template <typename T>
578  BatchSet<T> GetBatches(Context const* ctx, const BatchParam& param);
579  template <typename T>
580  [[nodiscard]] bool PageExists() const;
581 
587  [[nodiscard]] bool SingleColBlock() const { return this->NumBatches() == 1; }
588  [[nodiscard]] virtual std::int32_t NumBatches() const { return 1; }
589 
590  virtual ~DMatrix();
591 
595  [[nodiscard]] bool IsDense() const { return this->Info().IsDense(); }
596 
605  static DMatrix* Load(const std::string& uri, bool silent = true,
606  DataSplitMode data_split_mode = DataSplitMode::kRow);
607 
620  template <typename AdapterT>
621  static DMatrix* Create(AdapterT* adapter, float missing, int nthread,
622  const std::string& cache_prefix = "",
623  DataSplitMode data_split_mode = DataSplitMode::kRow);
624 
644  template <typename DataIterHandle, typename DMatrixHandle, typename DataIterResetCallback,
645  typename XGDMatrixCallbackNext>
646  static DMatrix* Create(DataIterHandle iter, DMatrixHandle proxy, std::shared_ptr<DMatrix> ref,
647  DataIterResetCallback* reset, XGDMatrixCallbackNext* next, float missing,
648  std::int32_t nthread, bst_bin_t max_bin, std::int64_t max_quantile_blocks);
649 
666  template <typename DataIterHandle, typename DMatrixHandle, typename DataIterResetCallback,
667  typename XGDMatrixCallbackNext>
669  XGDMatrixCallbackNext* next, ExtMemConfig const& config);
670 
678  template <typename DataIterHandle, typename DMatrixHandle, typename DataIterResetCallback,
679  typename XGDMatrixCallbackNext>
680  static DMatrix* Create(DataIterHandle iter, DMatrixHandle proxy, std::shared_ptr<DMatrix> ref,
682  bst_bin_t max_bin, std::int64_t max_quantile_blocks,
683  ExtMemConfig const& config);
684 
686 
694  virtual DMatrix *SliceCol(int num_slices, int slice_id) = 0;
695 
696  protected:
698  virtual BatchSet<CSCPage> GetColumnBatches(Context const* ctx) = 0;
700  virtual BatchSet<EllpackPage> GetEllpackBatches(Context const* ctx, BatchParam const& param) = 0;
702  BatchParam const& param) = 0;
703  virtual BatchSet<ExtSparsePage> GetExtBatches(Context const* ctx, BatchParam const& param) = 0;
704 
705  [[nodiscard]] virtual bool EllpackExists() const = 0;
706  [[nodiscard]] virtual bool GHistIndexExists() const = 0;
707  [[nodiscard]] virtual bool SparsePageExists() const = 0;
708 };
709 
710 template <>
712  return GetRowBatches();
713 }
714 
715 template <>
716 inline bool DMatrix::PageExists<EllpackPage>() const {
717  return this->EllpackExists();
718 }
719 
720 template <>
721 inline bool DMatrix::PageExists<GHistIndexMatrix>() const {
722  return this->GHistIndexExists();
723 }
724 
725 template <>
726 inline bool DMatrix::PageExists<SparsePage>() const {
727  return this->SparsePageExists();
728 }
729 
730 template <>
732  return GetRowBatches();
733 }
734 
735 template <>
736 inline BatchSet<CSCPage> DMatrix::GetBatches(Context const* ctx) {
737  return GetColumnBatches(ctx);
738 }
739 
740 template <>
741 inline BatchSet<SortedCSCPage> DMatrix::GetBatches(Context const* ctx) {
742  return GetSortedColumnBatches(ctx);
743 }
744 
745 template <>
747  return GetEllpackBatches(ctx, param);
748 }
749 
750 template <>
751 inline BatchSet<GHistIndexMatrix> DMatrix::GetBatches(Context const* ctx, BatchParam const& param) {
752  return GetGradientIndex(ctx, param);
753 }
754 
755 template <>
756 inline BatchSet<ExtSparsePage> DMatrix::GetBatches(Context const* ctx, BatchParam const& param) {
757  return GetExtBatches(ctx, param);
758 }
759 } // namespace xgboost
760 
762 
763 namespace dmlc {
765 
766 namespace serializer {
767 
768 template <>
769 struct Handler<xgboost::Entry> {
770  inline static void Write(Stream* strm, const xgboost::Entry& data) {
771  strm->Write(data.index);
772  strm->Write(data.fvalue);
773  }
774 
775  inline static bool Read(Stream* strm, xgboost::Entry* data) {
776  return strm->Read(&data->index) && strm->Read(&data->fvalue);
777  }
778 };
779 
780 } // namespace serializer
781 } // namespace dmlc
782 #endif // XGBOOST_DATA_H_
Defines configuration macros and basic types for xgboost.
#define XGBOOST_DEVICE
Tag function as usable by device.
Definition: base.h:64
Definition: data.h:459
virtual BatchIteratorImpl & operator++()=0
std::forward_iterator_tag iterator_category
Definition: data.h:461
virtual std::shared_ptr< T const > Page() const =0
virtual bool AtEnd() const =0
virtual const T & operator*() const =0
virtual ~BatchIteratorImpl()=default
Definition: data.h:470
BatchIterator(std::shared_ptr< BatchIteratorImpl< T >> impl)
Definition: data.h:474
std::forward_iterator_tag iterator_category
Definition: data.h:472
BatchIterator(BatchIteratorImpl< T > *impl)
Definition: data.h:473
const T & operator*() const
Definition: data.h:482
std::shared_ptr< T const > Page() const
Definition: data.h:494
BatchIterator & operator++()
Definition: data.h:476
bool operator!=(const BatchIterator &) const
Definition: data.h:487
bool AtEnd() const
Definition: data.h:489
Definition: data.h:503
BatchSet(BatchIterator< T > begin_iter)
Definition: data.h:505
BatchIterator< T > begin()
Definition: data.h:506
BatchIterator< T > end()
Definition: data.h:507
Definition: data.h:433
CSCPage()
Definition: data.h:435
CSCPage(SparsePage page)
Definition: data.h:436
Internal data structured used by XGBoost to hold all external data.
Definition: data.h:549
virtual BatchSet< EllpackPage > GetEllpackBatches(Context const *ctx, BatchParam const &param)=0
static DMatrix * Load(const std::string &uri, bool silent=true, DataSplitMode data_split_mode=DataSplitMode::kRow)
Load DMatrix from URI.
virtual BatchSet< SparsePage > GetRowBatches()=0
virtual BatchSet< GHistIndexMatrix > GetGradientIndex(Context const *ctx, BatchParam const &param)=0
static DMatrix * Create(DataIterHandle iter, DMatrixHandle proxy, DataIterResetCallback *reset, XGDMatrixCallbackNext *next, ExtMemConfig const &config)
Create an external memory DMatrix with callbacks.
virtual void SetInfo(const char *key, std::string const &interface_str)
Definition: data.h:555
virtual BatchSet< ExtSparsePage > GetExtBatches(Context const *ctx, BatchParam const &param)=0
bool PageExists() const
BatchSet< T > GetBatches(Context const *ctx)
virtual ~DMatrix()
virtual MetaInfo & Info()=0
meta information of the dataset
static DMatrix * Create(DataIterHandle iter, DMatrixHandle proxy, std::shared_ptr< DMatrix > ref, DataIterResetCallback *reset, XGDMatrixCallbackNext *next, float missing, std::int32_t nthread, bst_bin_t max_bin, std::int64_t max_quantile_blocks)
Create a new Quantile based DMatrix used for histogram based algorithm.
static DMatrix * Create(AdapterT *adapter, float missing, int nthread, const std::string &cache_prefix="", DataSplitMode data_split_mode=DataSplitMode::kRow)
Creates a new DMatrix from an external data adapter.
virtual DMatrix * SliceCol(int num_slices, int slice_id)=0
Slice a DMatrix by columns.
virtual bool GHistIndexExists() const =0
XGBAPIThreadLocalEntry & GetThreadLocal() const
Get thread local memory for returning data from DMatrix.
virtual bool SparsePageExists() const =0
virtual DMatrix * Slice(common::Span< int32_t const > ridxs)=0
virtual Context const * Ctx() const =0
Get the context object of this DMatrix. The context is created during construction of DMatrix with us...
BatchSet< T > GetBatches()
Gets batches. Use range based for loop over BatchSet to access individual batches.
virtual const MetaInfo & Info() const =0
meta information of the dataset
virtual bool EllpackExists() const =0
bool SingleColBlock() const
Definition: data.h:587
virtual BatchSet< CSCPage > GetColumnBatches(Context const *ctx)=0
virtual BatchSet< SortedCSCPage > GetSortedColumnBatches(Context const *ctx)=0
BatchSet< T > GetBatches(Context const *ctx, const BatchParam &param)
static DMatrix * Create(DataIterHandle iter, DMatrixHandle proxy, std::shared_ptr< DMatrix > ref, DataIterResetCallback *reset, XGDMatrixCallbackNext *next, bst_bin_t max_bin, std::int64_t max_quantile_blocks, ExtMemConfig const &config)
Create an external memory quantile DMatrix with callbacks.
virtual std::int32_t NumBatches() const
Definition: data.h:588
bool IsDense() const
Whether the matrix is dense.
Definition: data.h:595
DMatrix()=default
default constructor
Sparse page for exporting DMatrix. Same as SparsePage, just a different type to prevent being used in...
Definition: data.h:443
ExtSparsePage(std::shared_ptr< SparsePage const > p)
Definition: data.h:446
std::shared_ptr< SparsePage const > page
Definition: data.h:445
std::size_t Size() const
common::Span< T const > ConstHostSpan() const
Definition: host_device_vector.h:115
std::vector< T > & HostVector()
Data structure representing JSON format.
Definition: json.h:378
Meta information about dataset, always sit in memory.
Definition: data.h:48
std::vector< std::string > feature_names
Name for each feature.
Definition: data.h:92
MetaInfo(MetaInfo &&that)=default
HostDeviceVector< bst_float > labels_upper_bound_
upper bound of the label, to be used for survival analysis (censored regression)
Definition: data.h:83
MetaInfo Slice(Context const *ctx, common::Span< bst_idx_t const > ridxs, bst_idx_t nnz) const
Slice the meta info.
void SynchronizeNumberOfColumns(Context const *ctx, DataSplitMode split_mode)
Synchronize the number of columns across all workers.
uint64_t num_col_
number of columns in the data
Definition: data.h:56
std::vector< std::string > feature_type_names
Name of type for each feature provided by users. Eg. "int"/"float"/"i"/"q".
Definition: data.h:88
HostDeviceVector< bst_float > weights_
weights of each instance, optional
Definition: data.h:69
bool IsVerticalFederated() const
A convenient method to check if we are doing vertical federated learning, which requires some special...
MetaInfo & operator=(MetaInfo const &that)=delete
void GetInfo(char const *key, bst_ulong *out_len, DataType dtype, const void **out_dptr) const
bool IsColumnSplit() const
Whether the data is split column-wise.
Definition: data.h:187
bst_float GetWeight(size_t i) const
Get weight of each instances.
Definition: data.h:133
HostDeviceVector< FeatureType > feature_types
Definition: data.h:96
DataSplitMode data_split_mode
data split mode
Definition: data.h:62
void LoadBinary(dmlc::Stream *fi)
Load the Meta info from binary stream.
std::vector< bst_group_t > group_ptr_
the index of begin and end of a group needed when the learning task is ranking.
Definition: data.h:67
void Validate(DeviceOrd device) const
Validate all metainfo.
HostDeviceVector< float > feature_weights
Definition: data.h:101
void GetFeatureInfo(const char *field, std::vector< std::string > *out_str_vecs) const
bst_idx_t num_row_
number of rows in the data
Definition: data.h:54
bool HasCategorical() const
Flag for whether the DMatrix has categorical features.
Definition: data.h:207
MetaInfo & operator=(MetaInfo &&that)=default
bool IsDense() const
Whether the matrix is dense.
Definition: data.h:127
void Clear()
clear all the information
void Extend(MetaInfo const &that, bool accumulate_rows, bool check_column)
Extend with other MetaInfo.
bool IsRanking() const
Whether this is a learning to rank data.
Definition: data.h:189
uint64_t num_nonzero_
number of nonzero entries in the data
Definition: data.h:58
MetaInfo Copy() const
linalg::Tensor< float, 2 > labels
label of each instance
Definition: data.h:60
void SaveBinary(dmlc::Stream *fo) const
Save the Meta info to binary stream.
bool ShouldHaveLabels() const
A convenient method to check if the MetaInfo should contain labels.
MetaInfo()=default
default constructor
void SetInfo(Context const &ctx, StringView key, StringView interface_str)
Set information in the meta info with array interface.
static constexpr uint64_t kNumField
number of data fields in MetaInfo
Definition: data.h:51
bool IsRowSplit() const
Whether the data is split row-wise.
Definition: data.h:185
linalg::Matrix< float > base_margin_
initialized margins, if specified, xgboost will start from this init margin can be used to specify in...
Definition: data.h:75
HostDeviceVector< bst_float > labels_lower_bound_
lower bound of the label, to be used for survival analysis (censored regression)
Definition: data.h:79
const std::vector< size_t > & LabelAbsSort(Context const *ctx) const
get sorted indexes (argsort) of labels by absolute value (used by cox loss)
void SetFeatureInfo(const char *key, const char **info, const bst_ulong size)
Definition: data.h:449
SortedCSCPage(SparsePage page)
Definition: data.h:452
SortedCSCPage()
Definition: data.h:451
In-memory storage unit of sparse batch, stored in CSR format.
Definition: data.h:340
void Push(const SparsePage &batch)
Push a sparse page.
SparsePage()
constructor
Definition: data.h:357
SparsePage GetTranspose(int num_columns, int32_t n_threads) const
void SetBaseRowId(size_t row_id)
Set the base row id for this page.
Definition: data.h:387
void Reindex(uint64_t feature_offset, int32_t n_threads)
Reindex the column index with an offset.
uint64_t Push(const AdapterBatchT &batch, float missing, int nthread)
Pushes external data batch onto this page.
void PushCSC(const SparsePage &batch)
Push a SparsePage stored in CSC format.
bool IsIndicesSorted(int32_t n_threads) const
Check wether the column index is sorted.
virtual ~SparsePage()=default
void SortIndices(int32_t n_threads)
Sort the column index.
HostDeviceVector< Entry > data
the data of the segments
Definition: data.h:345
HostSparsePageView GetView() const
Definition: data.h:352
SparsePage & operator=(SparsePage const &that)=delete
size_t MemCostBytes() const
Definition: data.h:373
void Clear()
clear the page
Definition: data.h:378
SparsePage(SparsePage const &that)=delete
size_t Size() const
Definition: data.h:368
void SortRows(int32_t n_threads)
HostDeviceVector< bst_idx_t > offset
Definition: data.h:343
SparsePage & operator=(SparsePage &&that)=default
SparsePage(SparsePage &&that)=default
size_t base_rowid
Definition: data.h:347
span class implementation, based on ISO++20 span<T>. The interface should be the same.
Definition: span.h:431
constexpr XGBOOST_DEVICE pointer data() const __span_noexcept
Definition: span.h:550
std::size_t index_type
Definition: span.h:435
constexpr XGBOOST_DEVICE index_type size() const __span_noexcept
Definition: span.h:555
DECLARE_FIELD_ENUM_CLASS(xgboost::DataSplitMode)
void * DMatrixHandle
handle to DMatrix
Definition: c_api.h:50
int XGDMatrixCallbackNext(DataIterHandle iter)
Callback function prototype for getting next batch of data.
Definition: c_api.h:445
void * DataIterHandle
handle to a external data iterator
Definition: c_api.h:343
void DataIterResetCallback(DataIterHandle handle)
Callback function prototype for resetting the external iterator.
Definition: c_api.h:450
A device-and-host vector abstraction layer.
Linear algebra related utilities.
Definition: data.h:763
DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true)
Definition: intrusive_ptr.h:207
Core data structure for multi-target trees.
Definition: base.h:89
FeatureType
Definition: data.h:41
std::int32_t bst_bin_t
Type for histogram bin index. We sometimes use -1 to indicate invalid bin.
Definition: base.h:103
std::uint64_t bst_idx_t
Type for data row index (sample).
Definition: base.h:107
DataSplitMode
Definition: data.h:43
std::uint32_t bst_feature_t
Type for data column (feature) index.
Definition: base.h:99
DataType
data type accepted by xgboost interface
Definition: data.h:33
std::uint64_t bst_ulong
unsigned long integers
Definition: base.h:93
float bst_float
float type, used for storing statistics
Definition: base.h:95
static void Write(Stream *strm, const xgboost::Entry &data)
Definition: data.h:770
static bool Read(Stream *strm, xgboost::Entry *data)
Definition: data.h:775
Parameters for constructing histogram index batches.
Definition: data.h:247
bool forbid_regen
Forbid regenerating the gradient index. Used for internal validation.
Definition: data.h:264
bst_bin_t max_bin
Maximum number of bins per feature for histograms.
Definition: data.h:251
common::Span< float const > hess
Hessian, used for sketching with future approx implementation.
Definition: data.h:255
bool regen
Whether should we force DMatrix to regenerate the batch. Only used for GHistIndex.
Definition: data.h:260
bool ParamNotEqual(BatchParam const &other) const
Definition: data.h:298
BatchParam()=default
Exact or others that don't need histogram.
bool prefetch_copy
Used for GPU external memory. Whether to copy the data into device.
Definition: data.h:274
double sparse_thresh
Parameter used to generate column matrix for hist.
Definition: data.h:268
bool Initialized() const
Definition: data.h:309
BatchParam(bst_bin_t max_bin, common::Span< float const > hessian, bool regenerate)
Used by the approx tree method.
Definition: data.h:295
BatchParam MakeCache() const
Make a copy of self for DMatrix to describe how its existing index was generated.
Definition: data.h:313
BatchParam(bst_bin_t max_bin, double sparse_thresh)
Used by the hist tree method.
Definition: data.h:287
std::int32_t n_prefetch_batches
The number of batches to pre-fetch for external memory.
Definition: data.h:278
Runtime context for XGBoost. Contains information like threads and device.
Definition: context.h:133
A type for device ordinal. The type is packed into 32-bit for efficient use in viewing types like lin...
Definition: context.h:34
Element from a sparse vector.
Definition: data.h:219
XGBOOST_DEVICE Entry(bst_feature_t index, bst_float fvalue)
constructor with index and value
Definition: data.h:231
Entry()=default
default constructor
bst_feature_t index
feature index
Definition: data.h:221
static bool CmpIndex(Entry const &a, Entry const &b)
Definition: data.h:236
bst_float fvalue
feature value
Definition: data.h:223
bool operator==(const Entry &other) const
Definition: data.h:239
static bool CmpValue(const Entry &a, const Entry &b)
reversely compare feature values
Definition: data.h:233
Definition: data.h:516
ExtMemConfig(std::string cache, bool on_host, std::int64_t min_cache, float missing, std::int64_t max_num_d, std::int32_t n_threads)
Definition: data.h:532
std::int32_t n_threads
Definition: data.h:529
std::int64_t min_cache_page_bytes
Definition: data.h:523
bool on_host
Definition: data.h:520
float missing
Definition: data.h:525
std::string cache
Definition: data.h:518
std::int64_t max_num_device_pages
Definition: data.h:527
Definition: data.h:323
size_t Size() const
Definition: data.h:334
Inst operator[](std::size_t i) const
Definition: data.h:329
common::Span< bst_idx_t const > offset
Definition: data.h:326
common::Span< Entry const > data
Definition: data.h:327
Definition: string_view.h:16