xgboost

data.h
Go to the documentation of this file.
1
7 #ifndef XGBOOST_DATA_H_
8 #define XGBOOST_DATA_H_
9
10 #include <dmlc/base.h>
11 #include <dmlc/data.h>
12 #include <dmlc/serializer.h>
13 #include <xgboost/base.h>
14 #include <xgboost/generic_parameters.h>
15 #include <xgboost/host_device_vector.h>
16 #include <xgboost/linalg.h>
17 #include <xgboost/span.h>
18 #include <xgboost/string_view.h>
19
20 #include <algorithm>
21 #include <limits>
22 #include <memory>
23 #include <numeric>
24 #include <string>
25 #include <utility>
26 #include <vector>
27
28 namespace xgboost {
29 // forward declare dmatrix.
30 class DMatrix;
31
33 enum class DataType : uint8_t {
34 kFloat32 = 1,
35 kDouble = 2,
36 kUInt32 = 3,
37 kUInt64 = 4,
38 kStr = 5
39 };
40
41 enum class FeatureType : uint8_t { kNumerical = 0, kCategorical = 1 };
42
46 class MetaInfo {
47 public:
49 static constexpr uint64_t kNumField = 12;
50
52 uint64_t num_row_{0}; // NOLINT
54 uint64_t num_col_{0}; // NOLINT
56 uint64_t num_nonzero_{0}; // NOLINT
58 linalg::Tensor<float, 2> labels;
63 std::vector<bst_group_t> group_ptr_; // NOLINT
65 HostDeviceVector<bst_float> weights_; // NOLINT
71 linalg::Tensor<float, 2> base_margin_; // NOLINT
75 HostDeviceVector<bst_float> labels_lower_bound_; // NOLINT
79 HostDeviceVector<bst_float> labels_upper_bound_; // NOLINT
80
84 std::vector<std::string> feature_type_names;
88 std::vector<std::string> feature_names;
89 /*
90 * \brief Type of each feature. Automatically set when feature_type_names is specifed.
91 */
92 HostDeviceVector<FeatureType> feature_types;
93 /*
94 * \brief Weight of each feature, used to define the probability of each feature being
95 * selected when using column sampling.
96 */
97 HostDeviceVector<float> feature_weights;
98
100 MetaInfo() = default;
101 MetaInfo(MetaInfo&& that) = default;
102 MetaInfo& operator=(MetaInfo&& that) = default;
103 MetaInfo& operator=(MetaInfo const& that) = delete;
104
108 void Validate(int32_t device) const;
109
110 MetaInfo Slice(common::Span<int32_t const> ridxs) const;
116 inline bst_float GetWeight(size_t i) const {
117 return weights_.Size() != 0 ? weights_.HostVector()[i] : 1.0f;
118 }
120 inline const std::vector<size_t>& LabelAbsSort() const {
121 if (label_order_cache_.size() == labels.Size()) {
122 return label_order_cache_;
123 }
124 label_order_cache_.resize(labels.Size());
125 std::iota(label_order_cache_.begin(), label_order_cache_.end(), 0);
126 const auto& l = labels.Data()->HostVector();
127 XGBOOST_PARALLEL_STABLE_SORT(label_order_cache_.begin(), label_order_cache_.end(),
128 [&l](size_t i1, size_t i2) {return std::abs(l[i1]) < std::abs(l[i2]);});
129
130 return label_order_cache_;
131 }
133 void Clear();
138 void LoadBinary(dmlc::Stream* fi);
143 void SaveBinary(dmlc::Stream* fo) const;
151 void SetInfo(Context const& ctx, const char* key, const void* dptr, DataType dtype, size_t num);
157 void SetInfo(Context const& ctx, StringView key, StringView interface_str);
158
159 void GetInfo(char const* key, bst_ulong* out_len, DataType dtype,
160 const void** out_dptr) const;
161
162 void SetFeatureInfo(const char *key, const char **info, const bst_ulong size);
163 void GetFeatureInfo(const char *field, std::vector<std::string>* out_str_vecs) const;
164
165 /*
166 * \brief Extend with other MetaInfo.
167 *
168 * \param that The other MetaInfo object.
169 *
170 * \param accumulate_rows Whether rows need to be accumulated in this function. If
171 * client code knows number of rows in advance, set this
172 * parameter to false.
173 * \param check_column Whether the extend method should check the consistency of
174 * columns.
175 */
176 void Extend(MetaInfo const& that, bool accumulate_rows, bool check_column);
177
178 private:
179 void SetInfoFromHost(Context const& ctx, StringView key, Json arr);
180 void SetInfoFromCUDA(Context const& ctx, StringView key, Json arr);
181
183 mutable std::vector<size_t> label_order_cache_;
184 };
185
187 struct Entry {
189 bst_feature_t index;
191 bst_float fvalue;
193 Entry() = default;
199 XGBOOST_DEVICE Entry(bst_feature_t index, bst_float fvalue) : index(index), fvalue(fvalue) {}
201 inline static bool CmpValue(const Entry& a, const Entry& b) {
202 return a.fvalue < b.fvalue;
203 }
204 static bool CmpIndex(Entry const& a, Entry const& b) {
205 return a.index < b.index;
206 }
207 inline bool operator==(const Entry& other) const {
208 return (this->index == other.index && this->fvalue == other.fvalue);
209 }
210 };
211
215 struct BatchParam {
217 int gpu_id {-1};
219 int max_bin{0};
221 common::Span<float> hess;
223 bool regen {false};
225 double sparse_thresh{std::numeric_limits<double>::quiet_NaN()};
226
227 BatchParam() = default;
228 // GPU Hist
229 BatchParam(int32_t device, int32_t max_bin)
230 : gpu_id{device}, max_bin{max_bin} {}
231 // Hist
232 BatchParam(int32_t max_bin, double sparse_thresh)
233 : max_bin{max_bin}, sparse_thresh{sparse_thresh} {}
234 // Approx
239 BatchParam(int32_t max_bin, common::Span<float> hessian, bool regenerate)
240 : max_bin{max_bin}, hess{hessian}, regen{regenerate} {}
241
242 bool operator!=(BatchParam const& other) const {
243 if (hess.empty() && other.hess.empty()) {
244 return gpu_id != other.gpu_id || max_bin != other.max_bin;
245 }
246 return gpu_id != other.gpu_id || max_bin != other.max_bin || hess.data() != other.hess.data();
247 }
248 bool operator==(BatchParam const& other) const {
249 return !(*this != other);
250 }
251 };
252
253 struct HostSparsePageView {
254 using Inst = common::Span<Entry const>;
255
256 common::Span<bst_row_t const> offset;
257 common::Span<Entry const> data;
258
259 Inst operator[](size_t i) const {
260 auto size = *(offset.data() + i + 1) - *(offset.data() + i);
261 return {data.data() + *(offset.data() + i),
262 static_cast<Inst::index_type>(size)};
263 }
264
265 size_t Size() const { return offset.size() == 0 ? 0 : offset.size() - 1; }
266 };
267
271 class SparsePage {
272 public:
273 // Offset for each row.
274 HostDeviceVector<bst_row_t> offset;
276 HostDeviceVector<Entry> data;
277
278 size_t base_rowid {0};
279
281 using Inst = common::Span<Entry const>;
282
283 HostSparsePageView GetView() const {
284 return {offset.ConstHostSpan(), data.ConstHostSpan()};
285 }
286
287
289 SparsePage() {
290 this->Clear();
291 }
292
294 inline size_t Size() const {
295 return offset.Size() == 0 ? 0 : offset.Size() - 1;
296 }
297
299 inline size_t MemCostBytes() const {
300 return offset.Size() * sizeof(size_t) + data.Size() * sizeof(Entry);
301 }
302
304 inline void Clear() {
305 base_rowid = 0;
306 auto& offset_vec = offset.HostVector();
307 offset_vec.clear();
308 offset_vec.push_back(0);
309 data.HostVector().clear();
310 }
311
313 inline void SetBaseRowId(size_t row_id) {
314 base_rowid = row_id;
315 }
316
317 SparsePage GetTranspose(int num_columns, int32_t n_threads) const;
318
322 void SortIndices(int32_t n_threads);
326 bool IsIndicesSorted(int32_t n_threads) const;
327
328 void SortRows(int32_t n_threads);
329
340 template <typename AdapterBatchT>
341 uint64_t Push(const AdapterBatchT& batch, float missing, int nthread);
342
347 void Push(const SparsePage &batch);
352 void PushCSC(const SparsePage& batch);
353 };
354
355 class CSCPage: public SparsePage {
356 public:
357 CSCPage() : SparsePage() {}
358 explicit CSCPage(SparsePage page) : SparsePage(std::move(page)) {}
359 };
360
361 class SortedCSCPage : public SparsePage {
362 public:
363 SortedCSCPage() : SparsePage() {}
364 explicit SortedCSCPage(SparsePage page) : SparsePage(std::move(page)) {}
365 };
366
367 class EllpackPageImpl;
374 class EllpackPage {
375 public:
382 EllpackPage();
383
390 explicit EllpackPage(DMatrix* dmat, const BatchParam& param);
391
393 ~EllpackPage();
394
395 EllpackPage(EllpackPage&& that);
396
398 size_t Size() const;
399
401 void SetBaseRowId(size_t row_id);
402
403 const EllpackPageImpl* Impl() const { return impl_.get(); }
404 EllpackPageImpl* Impl() { return impl_.get(); }
405
406 private:
407 std::unique_ptr<EllpackPageImpl> impl_;
408 };
409
410 class GHistIndexMatrix;
411
412 template<typename T>
413 class BatchIteratorImpl {
414 public:
415 using iterator_category = std::forward_iterator_tag; // NOLINT
416 virtual ~BatchIteratorImpl() = default;
417 virtual const T& operator*() const = 0;
418 virtual BatchIteratorImpl& operator++() = 0;
419 virtual bool AtEnd() const = 0;
420 virtual std::shared_ptr<T const> Page() const = 0;
421 };
422
423 template<typename T>
424 class BatchIterator {
425 public:
426 using iterator_category = std::forward_iterator_tag; // NOLINT
427 explicit BatchIterator(BatchIteratorImpl<T>* impl) { impl_.reset(impl); }
428 explicit BatchIterator(std::shared_ptr<BatchIteratorImpl<T>> impl) { impl_ = impl; }
429
430 BatchIterator &operator++() {
431 CHECK(impl_ != nullptr);
432 ++(*impl_);
433 return *this;
434 }
435
436 const T& operator*() const {
437 CHECK(impl_ != nullptr);
438 return *(*impl_);
439 }
440
441 bool operator!=(const BatchIterator&) const {
442 CHECK(impl_ != nullptr);
443 return !impl_->AtEnd();
444 }
445
446 bool AtEnd() const {
447 CHECK(impl_ != nullptr);
448 return impl_->AtEnd();
449 }
450
451 std::shared_ptr<T const> Page() const {
452 return impl_->Page();
453 }
454
455 private:
456 std::shared_ptr<BatchIteratorImpl<T>> impl_;
457 };
458
459 template<typename T>
460 class BatchSet {
461 public:
462 explicit BatchSet(BatchIterator<T> begin_iter) : begin_iter_(std::move(begin_iter)) {}
463 BatchIterator<T> begin() { return begin_iter_; } // NOLINT
464 BatchIterator<T> end() { return BatchIterator<T>(nullptr); } // NOLINT
465
466 private:
467 BatchIterator<T> begin_iter_;
468 };
469
470 struct XGBAPIThreadLocalEntry;
471
475 class DMatrix {
476 public:
478 DMatrix() = default;
480 virtual MetaInfo& Info() = 0;
481 virtual void SetInfo(const char* key, const void* dptr, DataType dtype, size_t num) {
482 auto const& ctx = *this->Ctx();
483 this->Info().SetInfo(ctx, key, dptr, dtype, num);
484 }
485 virtual void SetInfo(const char* key, std::string const& interface_str) {
486 auto const& ctx = *this->Ctx();
487 this->Info().SetInfo(ctx, key, StringView{interface_str});
488 }
490 virtual const MetaInfo& Info() const = 0;
491
493 XGBAPIThreadLocalEntry& GetThreadLocal() const;
498 virtual Context const* Ctx() const = 0;
499
503 template <typename T>
504 BatchSet<T> GetBatches();
505 template <typename T>
506 BatchSet<T> GetBatches(const BatchParam& param);
507 template <typename T>
508 bool PageExists() const;
509
510 // the following are column meta data, should be able to answer them fast.
512 virtual bool SingleColBlock() const = 0;
514 virtual ~DMatrix();
515
517 bool IsDense() const {
518 return Info().num_nonzero_ == Info().num_row_ * Info().num_col_;
519 }
520
531 static DMatrix* Load(const std::string& uri,
532 bool silent,
533 bool load_row_split,
534 const std::string& file_format = "auto");
535
548 template <typename AdapterT>
549 static DMatrix* Create(AdapterT* adapter, float missing, int nthread,
550 const std::string& cache_prefix = "");
551
570 template <typename DataIterHandle, typename DMatrixHandle,
571 typename DataIterResetCallback, typename XGDMatrixCallbackNext>
572 static DMatrix *Create(DataIterHandle iter, DMatrixHandle proxy,
573 DataIterResetCallback *reset,
574 XGDMatrixCallbackNext *next, float missing,
575 int nthread,
576 int max_bin);
577
596 template <typename DataIterHandle, typename DMatrixHandle,
597 typename DataIterResetCallback, typename XGDMatrixCallbackNext>
598 static DMatrix *Create(DataIterHandle iter, DMatrixHandle proxy,
599 DataIterResetCallback *reset,
600 XGDMatrixCallbackNext *next, float missing,
601 int32_t nthread, std::string cache);
602
603 virtual DMatrix *Slice(common::Span<int32_t const> ridxs) = 0;
606 static const size_t kPageSize = 32UL << 12UL;
607
608 protected:
609 virtual BatchSet<SparsePage> GetRowBatches() = 0;
610 virtual BatchSet<CSCPage> GetColumnBatches() = 0;
611 virtual BatchSet<SortedCSCPage> GetSortedColumnBatches() = 0;
612 virtual BatchSet<EllpackPage> GetEllpackBatches(const BatchParam& param) = 0;
613 virtual BatchSet<GHistIndexMatrix> GetGradientIndex(const BatchParam& param) = 0;
614
615 virtual bool EllpackExists() const = 0;
616 virtual bool SparsePageExists() const = 0;
617 };
618
619 template<>
620 inline BatchSet<SparsePage> DMatrix::GetBatches() {
621 return GetRowBatches();
622 }
623
624 template<>
625 inline bool DMatrix::PageExists<EllpackPage>() const {
626 return this->EllpackExists();
627 }
628
629 template<>
630 inline bool DMatrix::PageExists<SparsePage>() const {
631 return this->SparsePageExists();
632 }
633
634 template<>
635 inline BatchSet<CSCPage> DMatrix::GetBatches() {
636 return GetColumnBatches();
637 }
638
639 template<>
640 inline BatchSet<SortedCSCPage> DMatrix::GetBatches() {
641 return GetSortedColumnBatches();
642 }
643
644 template<>
645 inline BatchSet<EllpackPage> DMatrix::GetBatches(const BatchParam& param) {
646 return GetEllpackBatches(param);
647 }
648
649 template<>
650 inline BatchSet<GHistIndexMatrix> DMatrix::GetBatches(const BatchParam& param) {
651 return GetGradientIndex(param);
652 }
653 } // namespace xgboost
654
655 namespace dmlc {
656 DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true);
657
658 namespace serializer {
659
660 template <>
661 struct Handler<xgboost::Entry> {
662 inline static void Write(Stream* strm, const xgboost::Entry& data) {
663 strm->Write(data.index);
664 strm->Write(data.fvalue);
665 }
666
667 inline static bool Read(Stream* strm, xgboost::Entry* data) {
668 return strm->Read(&data->index) && strm->Read(&data->fvalue);
669 }
670 };
671
672 } // namespace serializer
673 } // namespace dmlc
674 #endif // XGBOOST_DATA_H_
xgboost::MetaInfo::GetFeatureInfo
void GetFeatureInfo(const char *field, std::vector< std::string > *out_str_vecs) const
xgboost::Entry::index
bst_feature_t index
feature index
Definition: data.h:189
xgboost::BatchIteratorImpl::Page
virtual std::shared_ptr< T const > Page() const =0
xgboost::DMatrix::Info
virtual MetaInfo & Info()=0
meta information of the dataset
xgboost::BatchParam::max_bin
int max_bin
Maximum number of bins per feature for histograms.
Definition: data.h:219
xgboost::SparsePage::IsIndicesSorted
bool IsIndicesSorted(int32_t n_threads) const
Check wether the column index is sorted.
xgboost::MetaInfo::feature_type_names
std::vector< std::string > feature_type_names
Name of type for each feature provided by users. Eg. "int"/"float"/"i"/"q".
Definition: data.h:84
xgboost::DMatrix::GetEllpackBatches
virtual BatchSet< EllpackPage > GetEllpackBatches(const BatchParam &param)=0
xgboost::BatchIteratorImpl::operator++
virtual BatchIteratorImpl & operator++()=0
xgboost::MetaInfo::num_row_
uint64_t num_row_
number of rows in the data
Definition: data.h:52
xgboost::HostSparsePageView::data
common::Span< Entry const > data
Definition: data.h:257
xgboost::DMatrix::Ctx
virtual Context const * Ctx() const =0
Get the context object of this DMatrix. The context is created during construction of DMatrix with us...
xgboost::BatchIteratorImpl::~BatchIteratorImpl
virtual ~BatchIteratorImpl()=default
xgboost::SparsePage::Size
size_t Size() const
Definition: data.h:294
xgboost::SparsePage::SetBaseRowId
void SetBaseRowId(size_t row_id)
Set the base row id for this page.
Definition: data.h:313
xgboost::DataType::kUInt64
@ kUInt64
xgboost::MetaInfo::Extend
void Extend(MetaInfo const &that, bool accumulate_rows, bool check_column)
xgboost::CSCPage::CSCPage
CSCPage()
Definition: data.h:357
dmlc
Definition: data.h:655
xgboost::MetaInfo::LabelAbsSort
const std::vector< size_t > & LabelAbsSort() const
get sorted indexes (argsort) of labels by absolute value (used by cox loss)
Definition: data.h:120
xgboost::DMatrix::~DMatrix
virtual ~DMatrix()
virtual destructor
xgboost::EllpackPage
A page stored in ELLPACK format.
Definition: data.h:374
xgboost::SparsePage
In-memory storage unit of sparse batch, stored in CSR format.
Definition: data.h:271
xgboost::SparsePage::Push
uint64_t Push(const AdapterBatchT &batch, float missing, int nthread)
Pushes external data batch onto this page.
xgboost::CSCPage::CSCPage
CSCPage(SparsePage page)
Definition: data.h:358
xgboost::StringView
Definition: string_view.h:14
xgboost::common::Span::index_type
std::size_t index_type
Definition: span.h:427
xgboost::GenericParameter
Definition: generic_parameters.h:15
xgboost::Entry
Element from a sparse vector.
Definition: data.h:187
xgboost::MetaInfo::SetInfo
void SetInfo(Context const &ctx, const char *key, const void *dptr, DataType dtype, size_t num)
Set information in the meta info.
xgboost::BatchParam::operator!=
bool operator!=(BatchParam const &other) const
Definition: data.h:242
xgboost::EllpackPage::Impl
const EllpackPageImpl * Impl() const
Definition: data.h:403
xgboost::DMatrix::GetRowBatches
virtual BatchSet< SparsePage > GetRowBatches()=0
xgboost::HostDeviceVector< bst_float >
xgboost::BatchSet::BatchSet
BatchSet(BatchIterator< T > begin_iter)
Definition: data.h:462
xgboost::DMatrix::Create
static DMatrix * Create(AdapterT *adapter, float missing, int nthread, const std::string &cache_prefix="")
Creates a new DMatrix from an external data adapter.
host_device_vector.h
A device-and-host vector abstraction layer.
xgboost::DMatrix::SingleColBlock
virtual bool SingleColBlock() const =0
xgboost::SortedCSCPage::SortedCSCPage
SortedCSCPage(SparsePage page)
Definition: data.h:364
xgboost::MetaInfo::Clear
void Clear()
clear all the information
xgboost::BatchIterator::AtEnd
bool AtEnd() const
Definition: data.h:446
XGBOOST_PARALLEL_STABLE_SORT
#define XGBOOST_PARALLEL_STABLE_SORT(X, Y, Z)
Definition: base.h:69
xgboost::EllpackPage::Size
size_t Size() const
xgboost::SortedCSCPage::SortedCSCPage
SortedCSCPage()
Definition: data.h:363
xgboost::DMatrix::SetInfo
virtual void SetInfo(const char *key, std::string const &interface_str)
Definition: data.h:485
xgboost::BatchIterator::BatchIterator
BatchIterator(BatchIteratorImpl< T > *impl)
Definition: data.h:427
base.h
defines configuration macros of xgboost.
xgboost::MetaInfo::MetaInfo
MetaInfo()=default
default constructor
xgboost::MetaInfo::labels_lower_bound_
HostDeviceVector< bst_float > labels_lower_bound_
lower bound of the label, to be used for survival analysis (censored regression)
Definition: data.h:75
xgboost::HostDeviceVector::ConstHostSpan
common::Span< T const > ConstHostSpan() const
Definition: host_device_vector.h:114
xgboost::MetaInfo::Slice
MetaInfo Slice(common::Span< int32_t const > ridxs) const
xgboost::BatchIterator::Page
std::shared_ptr< T const > Page() const
Definition: data.h:451
xgboost::BatchIteratorImpl
Definition: data.h:413
DataIterHandle
void * DataIterHandle
handle to a external data iterator
Definition: c_api.h:296
xgboost::BatchSet::begin
BatchIterator< T > begin()
Definition: data.h:463
xgboost::BatchParam::BatchParam
BatchParam()=default
xgboost::MetaInfo::num_col_
uint64_t num_col_
number of columns in the data
Definition: data.h:54
xgboost::MetaInfo::SetFeatureInfo
void SetFeatureInfo(const char *key, const char **info, const bst_ulong size)
xgboost::MetaInfo::group_ptr_
std::vector< bst_group_t > group_ptr_
the index of begin and end of a group needed when the learning task is ranking.
Definition: data.h:63
xgboost::BatchIterator::operator*
const T & operator*() const
Definition: data.h:436
xgboost::DMatrix
Internal data structured used by XGBoost during training.
Definition: data.h:475
xgboost::MetaInfo::LoadBinary
void LoadBinary(dmlc::Stream *fi)
Load the Meta info from binary stream.
xgboost::DMatrix::Load
static DMatrix * Load(const std::string &uri, bool silent, bool load_row_split, const std::string &file_format="auto")
Load DMatrix from URI.
xgboost::MetaInfo::base_margin_
linalg::Tensor< float, 2 > base_margin_
initialized margins, if specified, xgboost will start from this init margin can be used to specify in...
Definition: data.h:71
XGDMatrixCallbackNext
XGB_EXTERN_C typedef int XGDMatrixCallbackNext(DataIterHandle iter)
Callback function prototype for getting next batch of data.
xgboost::BatchIterator::operator++
BatchIterator & operator++()
Definition: data.h:430
xgboost::BatchParam::BatchParam
BatchParam(int32_t device, int32_t max_bin)
Definition: data.h:229
xgboost::DataType::kDouble
@ kDouble
xgboost::MetaInfo::feature_names
std::vector< std::string > feature_names
Name for each feature.
Definition: data.h:88
xgboost::DataType::kUInt32
@ kUInt32
xgboost::bst_ulong
uint64_t bst_ulong
unsigned long integers
Definition: base.h:117
xgboost::HostSparsePageView::offset
common::Span< bst_row_t const > offset
Definition: data.h:256
xgboost::linalg::Tensor::Data
HostDeviceVector< T > * Data()
Definition: linalg.h:738
xgboost::DMatrix::GetGradientIndex
virtual BatchSet< GHistIndexMatrix > GetGradientIndex(const BatchParam &param)=0
xgboost::bst_feature_t
uint32_t bst_feature_t
Type for data column (feature) index.
Definition: base.h:123
xgboost::DMatrix::GetSortedColumnBatches
virtual BatchSet< SortedCSCPage > GetSortedColumnBatches()=0
xgboost::HostSparsePageView::operator[]
Inst operator[](size_t i) const
Definition: data.h:259
xgboost::common::Span::empty
constexpr XGBOOST_DEVICE bool empty() const __span_noexcept
Definition: span.h:560
xgboost::BatchIteratorImpl::iterator_category
std::forward_iterator_tag iterator_category
Definition: data.h:415
xgboost::DMatrix::DMatrix
DMatrix()=default
default constructor
xgboost::HostDeviceVector::HostVector
std::vector< T > & HostVector()
DataIterResetCallback
XGB_EXTERN_C typedef void DataIterResetCallback(DataIterHandle handle)
Callback function prototype for resetting external iterator.
xgboost::BatchParam::gpu_id
int gpu_id
The GPU device to use.
Definition: data.h:217
xgboost::SparsePage::GetTranspose
SparsePage GetTranspose(int num_columns, int32_t n_threads) const
xgboost::MetaInfo::feature_types
HostDeviceVector< FeatureType > feature_types
Definition: data.h:92
span.h
xgboost::DMatrix::EllpackExists
virtual bool EllpackExists() const =0
xgboost::BatchParam::BatchParam
BatchParam(int32_t max_bin, double sparse_thresh)
Definition: data.h:232
xgboost::Entry::Entry
XGBOOST_DEVICE Entry(bst_feature_t index, bst_float fvalue)
constructor with index and value
Definition: data.h:199
xgboost::EllpackPage::~EllpackPage
~EllpackPage()
Destructor.
xgboost::DMatrix::IsDense
bool IsDense() const
Whether the matrix is dense.
Definition: data.h:517
xgboost::SortedCSCPage
Definition: data.h:361
xgboost::Entry::operator==
bool operator==(const Entry &other) const
Definition: data.h:207
xgboost::BatchParam::operator==
bool operator==(BatchParam const &other) const
Definition: data.h:248
xgboost::BatchParam::sparse_thresh
double sparse_thresh
Parameter used to generate column matrix for hist.
Definition: data.h:225
xgboost::BatchIterator::operator!=
bool operator!=(const BatchIterator &) const
Definition: data.h:441
dmlc::serializer::Handler< xgboost::Entry >::Write
static void Write(Stream *strm, const xgboost::Entry &data)
Definition: data.h:662
xgboost::MetaInfo::GetWeight
bst_float GetWeight(size_t i) const
Get weight of each instances.
Definition: data.h:116
xgboost::DMatrix::GetBatches
BatchSet< T > GetBatches()
Gets batches. Use range based for loop over BatchSet to access individual batches.
xgboost::HostSparsePageView::Size
size_t Size() const
Definition: data.h:265
xgboost::SparsePage::GetView
HostSparsePageView GetView() const
Definition: data.h:283
linalg.h
Linear algebra related utilities.
xgboost::BatchParam
Parameters for constructing batches.
Definition: data.h:215
xgboost::SparsePage::base_rowid
size_t base_rowid
Definition: data.h:278
xgboost::HostSparsePageView
Definition: data.h:253
xgboost::Entry::CmpIndex
static bool CmpIndex(Entry const &a, Entry const &b)
Definition: data.h:204
xgboost::DataType
DataType
data type accepted by xgboost interface
Definition: data.h:33
xgboost::DMatrix::SparsePageExists
virtual bool SparsePageExists() const =0
xgboost::FeatureType::kNumerical
@ kNumerical
xgboost::DMatrix::kPageSize
static const size_t kPageSize
Number of rows per page in external memory. Approximately 100MB per page for dataset with 100 feature...
Definition: data.h:606
xgboost::SparsePage::MemCostBytes
size_t MemCostBytes() const
Definition: data.h:299
xgboost::BatchParam::BatchParam
BatchParam(int32_t max_bin, common::Span< float > hessian, bool regenerate)
Get batch with sketch weighted by hessian. The batch will be regenerated if the span is changed,...
Definition: data.h:239
xgboost::linalg::Tensor< float, 2 >
xgboost::SparsePage::PushCSC
void PushCSC(const SparsePage &batch)
Push a SparsePage stored in CSC format.
xgboost::MetaInfo::Validate
void Validate(int32_t device) const
Validate all metainfo.
xgboost::EllpackPage::Impl
EllpackPageImpl * Impl()
Definition: data.h:404
xgboost::HostDeviceVector::Size
size_t Size() const
string_view.h
xgboost::FeatureType
FeatureType
Definition: data.h:41
xgboost::CSCPage
Definition: data.h:355
xgboost::Entry::Entry
Entry()=default
default constructor
xgboost::BatchIteratorImpl::AtEnd
virtual bool AtEnd() const =0
xgboost::BatchSet::end
BatchIterator< T > end()
Definition: data.h:464
xgboost::DataType::kStr
@ kStr
xgboost::DMatrix::GetColumnBatches
virtual BatchSet< CSCPage > GetColumnBatches()=0
xgboost::MetaInfo::weights_
HostDeviceVector< bst_float > weights_
weights of each instance, optional
Definition: data.h:65
xgboost::EllpackPage::EllpackPage
EllpackPage()
Default constructor.
xgboost::MetaInfo::num_nonzero_
uint64_t num_nonzero_
number of nonzero entries in the data
Definition: data.h:56
xgboost::XGBAPIThreadLocalEntry
entry to to easily hold returning information
Definition: learner.h:45
xgboost::DataType::kFloat32
@ kFloat32
generic_parameters.h
xgboost::DMatrix::Slice
virtual DMatrix * Slice(common::Span< int32_t const > ridxs)=0
xgboost::common::Span
span class implementation, based on ISO++20 span<T>. The interface should be the same.
Definition: span.h:148
xgboost::SparsePage::SortRows
void SortRows(int32_t n_threads)
std
Definition: intrusive_ptr.h:207
xgboost::MetaInfo::operator=
MetaInfo & operator=(MetaInfo &&that)=default
xgboost::MetaInfo::SaveBinary
void SaveBinary(dmlc::Stream *fo) const
Save the Meta info to binary stream.
xgboost::BatchIterator::BatchIterator
BatchIterator(std::shared_ptr< BatchIteratorImpl< T >> impl)
Definition: data.h:428
xgboost::SparsePage::offset
HostDeviceVector< bst_row_t > offset
Definition: data.h:274
xgboost::MetaInfo::GetInfo
void GetInfo(char const *key, bst_ulong *out_len, DataType dtype, const void **out_dptr) const
DMatrixHandle
void * DMatrixHandle
handle to DMatrix
Definition: c_api.h:30
xgboost::common::Span::size
constexpr XGBOOST_DEVICE index_type size() const __span_noexcept
Definition: span.h:553
xgboost::BatchIterator
Definition: data.h:424
xgboost::common::Span::data
constexpr XGBOOST_DEVICE pointer data() const __span_noexcept
Definition: span.h:548
xgboost::Entry::CmpValue
static bool CmpValue(const Entry &a, const Entry &b)
reversely compare feature values
Definition: data.h:201
xgboost::MetaInfo
Meta information about dataset, always sit in memory.
Definition: data.h:46
xgboost::DMatrix::GetThreadLocal
XGBAPIThreadLocalEntry & GetThreadLocal() const
Get thread local memory for returning data from DMatrix.
xgboost::EllpackPage::SetBaseRowId
void SetBaseRowId(size_t row_id)
Set the base row id for this page.
xgboost::BatchIteratorImpl::operator*
virtual const T & operator*() const =0
xgboost::MetaInfo::labels
linalg::Tensor< float, 2 > labels
label of each instance
Definition: data.h:58
xgboost::Json
Data structure representing JSON format.
Definition: json.h:352
dmlc::DMLC_DECLARE_TRAITS
DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true)
xgboost::linalg::Tensor::Size
size_t Size() const
Definition: linalg.h:734
xgboost::Entry::fvalue
bst_float fvalue
feature value
Definition: data.h:191
xgboost::FeatureType::kCategorical
@ kCategorical
XGBOOST_DEVICE
#define XGBOOST_DEVICE
Tag function as usable by device.
Definition: base.h:84
xgboost::DMatrix::SetInfo
virtual void SetInfo(const char *key, const void *dptr, DataType dtype, size_t num)
Definition: data.h:481
xgboost::DMatrix::PageExists
bool PageExists() const
xgboost::SparsePage::data
HostDeviceVector< Entry > data
the data of the segments
Definition: data.h:276
xgboost::MetaInfo::feature_weights
HostDeviceVector< float > feature_weights
Definition: data.h:97
dmlc::serializer::Handler< xgboost::Entry >::Read
static bool Read(Stream *strm, xgboost::Entry *data)
Definition: data.h:667
xgboost::BatchIterator::iterator_category
std::forward_iterator_tag iterator_category
Definition: data.h:426
xgboost::BatchParam::hess
common::Span< float > hess
Hessian, used for sketching with future approx implementation.
Definition: data.h:221
xgboost::MetaInfo::kNumField
static constexpr uint64_t kNumField
number of data fields in MetaInfo
Definition: data.h:49
xgboost::BatchSet
Definition: data.h:460
xgboost
namespace of xgboost
Definition: base.h:110
xgboost::SparsePage::Clear
void Clear()
clear the page
Definition: data.h:304
xgboost::MetaInfo::labels_upper_bound_
HostDeviceVector< bst_float > labels_upper_bound_
upper bound of the label, to be used for survival analysis (censored regression)
Definition: data.h:79
xgboost::bst_float
float bst_float
float type, used for storing statistics
Definition: base.h:119
xgboost::SparsePage::SortIndices
void SortIndices(int32_t n_threads)
Sort the column index.
xgboost::SparsePage::SparsePage
SparsePage()
constructor
Definition: data.h:289
xgboost::BatchParam::regen
bool regen
Whether should DMatrix regenerate the batch. Only used for GHistIndex.
Definition: data.h:223