xgboost

data.h
Go to the documentation of this file.
1
7 #ifndef XGBOOST_DATA_H_
8 #define XGBOOST_DATA_H_
9
10 #include <dmlc/base.h>
11 #include <dmlc/data.h>
12 #include <dmlc/serializer.h>
13 #include <xgboost/base.h>
14 #include <xgboost/span.h>
15 #include <xgboost/host_device_vector.h>
16
17 #include <memory>
18 #include <numeric>
19 #include <algorithm>
20 #include <string>
21 #include <utility>
22 #include <vector>
23
24 namespace xgboost {
25 // forward declare dmatrix.
26 class DMatrix;
27
29 enum class DataType : uint8_t {
30 kFloat32 = 1,
31 kDouble = 2,
32 kUInt32 = 3,
33 kUInt64 = 4,
34 kStr = 5
35 };
36
37 enum class FeatureType : uint8_t {
38 kNumerical,
39 kCategorical
40 };
41
45 class MetaInfo {
46 public:
48 static constexpr uint64_t kNumField = 11;
49
51 uint64_t num_row_{0}; // NOLINT
53 uint64_t num_col_{0}; // NOLINT
55 uint64_t num_nonzero_{0}; // NOLINT
57 HostDeviceVector<bst_float> labels_; // NOLINT
62 std::vector<bst_group_t> group_ptr_; // NOLINT
64 HostDeviceVector<bst_float> weights_; // NOLINT
70 HostDeviceVector<bst_float> base_margin_; // NOLINT
74 HostDeviceVector<bst_float> labels_lower_bound_; // NOLINT
78 HostDeviceVector<bst_float> labels_upper_bound_; // NOLINT
79
83 std::vector<std::string> feature_type_names;
87 std::vector<std::string> feature_names;
88 /*
89 * \brief Type of each feature. Automatically set when feature_type_names is specifed.
90 */
91 HostDeviceVector<FeatureType> feature_types;
92 /*
93 * \brief Weight of each feature, used to define the probability of each feature being
94 * selected when using column sampling.
95 */
96 HostDeviceVector<float> feature_weigths;
97
99 MetaInfo() = default;
100 MetaInfo(MetaInfo&& that) = default;
101 MetaInfo& operator=(MetaInfo&& that) = default;
102 MetaInfo& operator=(MetaInfo const& that) = delete;
103
107 void Validate(int32_t device) const;
108
109 MetaInfo Slice(common::Span<int32_t const> ridxs) const;
115 inline bst_float GetWeight(size_t i) const {
116 return weights_.Size() != 0 ? weights_.HostVector()[i] : 1.0f;
117 }
119 inline const std::vector<size_t>& LabelAbsSort() const {
120 if (label_order_cache_.size() == labels_.Size()) {
121 return label_order_cache_;
122 }
123 label_order_cache_.resize(labels_.Size());
124 std::iota(label_order_cache_.begin(), label_order_cache_.end(), 0);
125 const auto& l = labels_.HostVector();
126 XGBOOST_PARALLEL_SORT(label_order_cache_.begin(), label_order_cache_.end(),
127 [&l](size_t i1, size_t i2) {return std::abs(l[i1]) < std::abs(l[i2]);});
128
129 return label_order_cache_;
130 }
132 void Clear();
137 void LoadBinary(dmlc::Stream* fi);
142 void SaveBinary(dmlc::Stream* fo) const;
150 void SetInfo(const char* key, const void* dptr, DataType dtype, size_t num);
160 void SetInfo(const char* key, std::string const& interface_str);
161
162 void GetInfo(char const* key, bst_ulong* out_len, DataType dtype,
163 const void** out_dptr) const;
164
165 void SetFeatureInfo(const char *key, const char **info, const bst_ulong size);
166 void GetFeatureInfo(const char *field, std::vector<std::string>* out_str_vecs) const;
167
168 /*
169 * \brief Extend with other MetaInfo.
170 *
171 * \param that The other MetaInfo object.
172 *
173 * \param accumulate_rows Whether rows need to be accumulated in this function. If
174 * client code knows number of rows in advance, set this parameter to false.
175 */
176 void Extend(MetaInfo const& that, bool accumulate_rows);
177
178 private:
180 mutable std::vector<size_t> label_order_cache_;
181 };
182
184 struct Entry {
186 bst_feature_t index;
188 bst_float fvalue;
190 Entry() = default;
196 XGBOOST_DEVICE Entry(bst_feature_t index, bst_float fvalue) : index(index), fvalue(fvalue) {}
198 inline static bool CmpValue(const Entry& a, const Entry& b) {
199 return a.fvalue < b.fvalue;
200 }
201 inline bool operator==(const Entry& other) const {
202 return (this->index == other.index && this->fvalue == other.fvalue);
203 }
204 };
205
209 struct BatchParam {
211 int gpu_id;
213 int max_bin{0};
215 size_t gpu_page_size;
216 BatchParam() = default;
217 BatchParam(int32_t device, int32_t max_bin, size_t gpu_page_size = 0)
218 : gpu_id{device}, max_bin{max_bin}, gpu_page_size{gpu_page_size} {}
219 inline bool operator!=(const BatchParam& other) const {
220 return gpu_id != other.gpu_id || max_bin != other.max_bin ||
221 gpu_page_size != other.gpu_page_size;
222 }
223 };
224
225 struct HostSparsePageView {
226 using Inst = common::Span<Entry const>;
227
228 common::Span<bst_row_t const> offset;
229 common::Span<Entry const> data;
230
231 Inst operator[](size_t i) const {
232 auto size = *(offset.data() + i + 1) - *(offset.data() + i);
233 return {data.data() + *(offset.data() + i),
234 static_cast<Inst::index_type>(size)};
235 }
236
237 size_t Size() const { return offset.size() == 0 ? 0 : offset.size() - 1; }
238 };
239
243 class SparsePage {
244 public:
245 // Offset for each row.
246 HostDeviceVector<bst_row_t> offset;
248 HostDeviceVector<Entry> data;
249
250 size_t base_rowid {0};
251
253 using Inst = common::Span<Entry const>;
254
256 inline Inst operator[](size_t i) const {
257 const auto& data_vec = data.HostVector();
258 const auto& offset_vec = offset.HostVector();
259 size_t size = offset_vec[i + 1] - offset_vec[i];
260 return {data_vec.data() + offset_vec[i],
261 static_cast<Inst::index_type>(size)};
262 }
263
264 HostSparsePageView GetView() const {
265 return {offset.ConstHostSpan(), data.ConstHostSpan()};
266 }
267
268
270 SparsePage() {
271 this->Clear();
272 }
273
275 inline size_t Size() const {
276 return offset.Size() == 0 ? 0 : offset.Size() - 1;
277 }
278
280 inline size_t MemCostBytes() const {
281 return offset.Size() * sizeof(size_t) + data.Size() * sizeof(Entry);
282 }
283
285 inline void Clear() {
286 base_rowid = 0;
287 auto& offset_vec = offset.HostVector();
288 offset_vec.clear();
289 offset_vec.push_back(0);
290 data.HostVector().clear();
291 }
292
294 inline void SetBaseRowId(size_t row_id) {
295 base_rowid = row_id;
296 }
297
298 SparsePage GetTranspose(int num_columns) const;
299
300 void SortRows() {
301 auto ncol = static_cast<bst_omp_uint>(this->Size());
302 #pragma omp parallel for default(none) shared(ncol) schedule(dynamic, 1)
303 for (bst_omp_uint i = 0; i < ncol; ++i) {
304 if (this->offset.HostVector()[i] < this->offset.HostVector()[i + 1]) {
305 std::sort(
306 this->data.HostVector().begin() + this->offset.HostVector()[i],
307 this->data.HostVector().begin() + this->offset.HostVector()[i + 1],
308 Entry::CmpValue);
309 }
310 }
311 }
312
323 template <typename AdapterBatchT>
324 uint64_t Push(const AdapterBatchT& batch, float missing, int nthread);
325
330 void Push(const SparsePage &batch);
335 void PushCSC(const SparsePage& batch);
336 };
337
338 class CSCPage: public SparsePage {
339 public:
340 CSCPage() : SparsePage() {}
341 explicit CSCPage(SparsePage page) : SparsePage(std::move(page)) {}
342 };
343
344 class SortedCSCPage : public SparsePage {
345 public:
346 SortedCSCPage() : SparsePage() {}
347 explicit SortedCSCPage(SparsePage page) : SparsePage(std::move(page)) {}
348 };
349
350 class EllpackPageImpl;
357 class EllpackPage {
358 public:
365 EllpackPage();
366
373 explicit EllpackPage(DMatrix* dmat, const BatchParam& param);
374
376 ~EllpackPage();
377
378 EllpackPage(EllpackPage&& that);
379
381 size_t Size() const;
382
384 void SetBaseRowId(size_t row_id);
385
386 const EllpackPageImpl* Impl() const { return impl_.get(); }
387 EllpackPageImpl* Impl() { return impl_.get(); }
388
389 private:
390 std::unique_ptr<EllpackPageImpl> impl_;
391 };
392
393 template<typename T>
394 class BatchIteratorImpl {
395 public:
396 virtual ~BatchIteratorImpl() = default;
397 virtual T& operator*() = 0;
398 virtual const T& operator*() const = 0;
399 virtual void operator++() = 0;
400 virtual bool AtEnd() const = 0;
401 };
402
403 template<typename T>
404 class BatchIterator {
405 public:
406 using iterator_category = std::forward_iterator_tag; // NOLINT
407 explicit BatchIterator(BatchIteratorImpl<T>* impl) { impl_.reset(impl); }
408
409 void operator++() {
410 CHECK(impl_ != nullptr);
411 ++(*impl_);
412 }
413
414 T& operator*() {
415 CHECK(impl_ != nullptr);
416 return *(*impl_);
417 }
418
419 const T& operator*() const {
420 CHECK(impl_ != nullptr);
421 return *(*impl_);
422 }
423
424 bool operator!=(const BatchIterator&) const {
425 CHECK(impl_ != nullptr);
426 return !impl_->AtEnd();
427 }
428
429 bool AtEnd() const {
430 CHECK(impl_ != nullptr);
431 return impl_->AtEnd();
432 }
433
434 private:
435 std::shared_ptr<BatchIteratorImpl<T>> impl_;
436 };
437
438 template<typename T>
439 class BatchSet {
440 public:
441 explicit BatchSet(BatchIterator<T> begin_iter) : begin_iter_(std::move(begin_iter)) {}
442 BatchIterator<T> begin() { return begin_iter_; } // NOLINT
443 BatchIterator<T> end() { return BatchIterator<T>(nullptr); } // NOLINT
444
445 private:
446 BatchIterator<T> begin_iter_;
447 };
448
449 struct XGBAPIThreadLocalEntry;
450
454 class DMatrix {
455 public:
457 DMatrix() = default;
459 virtual MetaInfo& Info() = 0;
460 virtual void SetInfo(const char *key, const void *dptr, DataType dtype,
461 size_t num) {
462 this->Info().SetInfo(key, dptr, dtype, num);
463 }
464 virtual void SetInfo(const char* key, std::string const& interface_str) {
465 this->Info().SetInfo(key, interface_str);
466 }
468 virtual const MetaInfo& Info() const = 0;
469
471 XGBAPIThreadLocalEntry& GetThreadLocal() const;
472
476 template<typename T>
477 BatchSet<T> GetBatches(const BatchParam& param = {});
478 template <typename T>
479 bool PageExists() const;
480
481 // the following are column meta data, should be able to answer them fast.
483 virtual bool SingleColBlock() const = 0;
485 virtual ~DMatrix();
486
488 bool IsDense() const {
489 return Info().num_nonzero_ == Info().num_row_ * Info().num_col_;
490 }
491
502 static DMatrix* Load(const std::string& uri,
503 bool silent,
504 bool load_row_split,
505 const std::string& file_format = "auto",
506 size_t page_size = kPageSize);
507
520 template <typename AdapterT>
521 static DMatrix* Create(AdapterT* adapter, float missing, int nthread,
522 const std::string& cache_prefix = "",
523 size_t page_size = kPageSize);
524
543 template <typename DataIterHandle, typename DMatrixHandle,
544 typename DataIterResetCallback, typename XGDMatrixCallbackNext>
545 static DMatrix *Create(DataIterHandle iter, DMatrixHandle proxy,
546 DataIterResetCallback *reset,
547 XGDMatrixCallbackNext *next, float missing,
548 int nthread,
549 int max_bin);
550
551 virtual DMatrix *Slice(common::Span<int32_t const> ridxs) = 0;
554 static const size_t kPageSize = 32UL << 12UL;
555
556 protected:
557 virtual BatchSet<SparsePage> GetRowBatches() = 0;
558 virtual BatchSet<CSCPage> GetColumnBatches() = 0;
559 virtual BatchSet<SortedCSCPage> GetSortedColumnBatches() = 0;
560 virtual BatchSet<EllpackPage> GetEllpackBatches(const BatchParam& param) = 0;
561
562 virtual bool EllpackExists() const = 0;
563 virtual bool SparsePageExists() const = 0;
564 };
565
566 template<>
567 inline BatchSet<SparsePage> DMatrix::GetBatches(const BatchParam&) {
568 return GetRowBatches();
569 }
570
571 template<>
572 inline bool DMatrix::PageExists<EllpackPage>() const {
573 return this->EllpackExists();
574 }
575
576 template<>
577 inline bool DMatrix::PageExists<SparsePage>() const {
578 return this->SparsePageExists();
579 }
580
581 template<>
582 inline BatchSet<CSCPage> DMatrix::GetBatches(const BatchParam&) {
583 return GetColumnBatches();
584 }
585
586 template<>
587 inline BatchSet<SortedCSCPage> DMatrix::GetBatches(const BatchParam&) {
588 return GetSortedColumnBatches();
589 }
590
591 template<>
592 inline BatchSet<EllpackPage> DMatrix::GetBatches(const BatchParam& param) {
593 return GetEllpackBatches(param);
594 }
595 } // namespace xgboost
596
597 namespace dmlc {
598 DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true);
599
600 namespace serializer {
601
602 template <>
603 struct Handler<xgboost::Entry> {
604 inline static void Write(Stream* strm, const xgboost::Entry& data) {
605 strm->Write(data.index);
606 strm->Write(data.fvalue);
607 }
608
609 inline static bool Read(Stream* strm, xgboost::Entry* data) {
610 return strm->Read(&data->index) && strm->Read(&data->fvalue);
611 }
612 };
613
614 } // namespace serializer
615 } // namespace dmlc
616 #endif // XGBOOST_DATA_H_
xgboost::MetaInfo::GetFeatureInfo
void GetFeatureInfo(const char *field, std::vector< std::string > *out_str_vecs) const
xgboost::BatchIteratorImpl::operator++
virtual void operator++()=0
xgboost::Entry::index
bst_feature_t index
feature index
Definition: data.h:186
xgboost::DMatrix::Info
virtual MetaInfo & Info()=0
meta information of the dataset
xgboost::BatchParam::max_bin
int max_bin
Maximum number of bins per feature for histograms.
Definition: data.h:213
xgboost::MetaInfo::feature_type_names
std::vector< std::string > feature_type_names
Name of type for each feature provided by users. Eg. "int"/"float"/"i"/"q".
Definition: data.h:83
xgboost::DMatrix::GetEllpackBatches
virtual BatchSet< EllpackPage > GetEllpackBatches(const BatchParam &param)=0
xgboost::MetaInfo::num_row_
uint64_t num_row_
number of rows in the data
Definition: data.h:51
xgboost::HostSparsePageView::data
common::Span< Entry const > data
Definition: data.h:229
xgboost::BatchIteratorImpl::~BatchIteratorImpl
virtual ~BatchIteratorImpl()=default
xgboost::SparsePage::Size
size_t Size() const
Definition: data.h:275
xgboost::SparsePage::SetBaseRowId
void SetBaseRowId(size_t row_id)
Set the base row id for this page.
Definition: data.h:294
xgboost::DataType::kUInt64
@ kUInt64
xgboost::CSCPage::CSCPage
CSCPage()
Definition: data.h:340
dmlc
Definition: data.h:597
xgboost::MetaInfo::LabelAbsSort
const std::vector< size_t > & LabelAbsSort() const
get sorted indexes (argsort) of labels by absolute value (used by cox loss)
Definition: data.h:119
xgboost::DMatrix::~DMatrix
virtual ~DMatrix()
virtual destructor
xgboost::EllpackPage
A page stored in ELLPACK format.
Definition: data.h:357
xgboost::SparsePage
In-memory storage unit of sparse batch, stored in CSR format.
Definition: data.h:243
xgboost::SparsePage::Push
uint64_t Push(const AdapterBatchT &batch, float missing, int nthread)
Pushes external data batch onto this page.
xgboost::CSCPage::CSCPage
CSCPage(SparsePage page)
Definition: data.h:341
xgboost::common::Span::index_type
std::size_t index_type
Definition: span.h:416
xgboost::SparsePage::SortRows
void SortRows()
Definition: data.h:300
xgboost::Entry
Element from a sparse vector.
Definition: data.h:184
xgboost::BatchIterator::operator++
void operator++()
Definition: data.h:409
xgboost::MetaInfo::base_margin_
HostDeviceVector< bst_float > base_margin_
initialized margins, if specified, xgboost will start from this init margin can be used to specify in...
Definition: data.h:70
xgboost::EllpackPage::Impl
const EllpackPageImpl * Impl() const
Definition: data.h:386
xgboost::DMatrix::GetRowBatches
virtual BatchSet< SparsePage > GetRowBatches()=0
xgboost::HostDeviceVector< bst_float >
xgboost::BatchSet::BatchSet
BatchSet(BatchIterator< T > begin_iter)
Definition: data.h:441
host_device_vector.h
A device-and-host vector abstraction layer.
xgboost::DMatrix::SingleColBlock
virtual bool SingleColBlock() const =0
xgboost::SortedCSCPage::SortedCSCPage
SortedCSCPage(SparsePage page)
Definition: data.h:347
xgboost::MetaInfo::Clear
void Clear()
clear all the information
xgboost::BatchIterator::AtEnd
bool AtEnd() const
Definition: data.h:429
xgboost::EllpackPage::Size
size_t Size() const
xgboost::SortedCSCPage::SortedCSCPage
SortedCSCPage()
Definition: data.h:346
xgboost::DMatrix::SetInfo
virtual void SetInfo(const char *key, std::string const &interface_str)
Definition: data.h:464
xgboost::BatchIterator::BatchIterator
BatchIterator(BatchIteratorImpl< T > *impl)
Definition: data.h:407
xgboost::MetaInfo::SetInfo
void SetInfo(const char *key, const void *dptr, DataType dtype, size_t num)
Set information in the meta info.
base.h
defines configuration macros of xgboost.
xgboost::MetaInfo::MetaInfo
MetaInfo()=default
default constructor
xgboost::BatchParam::operator!=
bool operator!=(const BatchParam &other) const
Definition: data.h:219
xgboost::MetaInfo::labels_lower_bound_
HostDeviceVector< bst_float > labels_lower_bound_
lower bound of the label, to be used for survival analysis (censored regression)
Definition: data.h:74
xgboost::HostDeviceVector::ConstHostSpan
common::Span< T const > ConstHostSpan() const
Definition: host_device_vector.h:114
xgboost::MetaInfo::Slice
MetaInfo Slice(common::Span< int32_t const > ridxs) const
xgboost::BatchIteratorImpl
Definition: data.h:394
DataIterHandle
void * DataIterHandle
handle to a external data iterator
Definition: c_api.h:191
xgboost::BatchSet::begin
BatchIterator< T > begin()
Definition: data.h:442
xgboost::BatchParam::BatchParam
BatchParam()=default
xgboost::MetaInfo::num_col_
uint64_t num_col_
number of columns in the data
Definition: data.h:53
xgboost::MetaInfo::SetFeatureInfo
void SetFeatureInfo(const char *key, const char **info, const bst_ulong size)
xgboost::MetaInfo::group_ptr_
std::vector< bst_group_t > group_ptr_
the index of begin and end of a group needed when the learning task is ranking.
Definition: data.h:62
xgboost::BatchIterator::operator*
const T & operator*() const
Definition: data.h:419
xgboost::BatchIterator::operator*
T & operator*()
Definition: data.h:414
xgboost::DMatrix
Internal data structured used by XGBoost during training.
Definition: data.h:454
xgboost::MetaInfo::LoadBinary
void LoadBinary(dmlc::Stream *fi)
Load the Meta info from binary stream.
XGDMatrixCallbackNext
XGB_EXTERN_C typedef int XGDMatrixCallbackNext(DataIterHandle iter)
Callback function prototype for getting next batch of data.
xgboost::DataType::kDouble
@ kDouble
xgboost::MetaInfo::feature_names
std::vector< std::string > feature_names
Name for each feature.
Definition: data.h:87
xgboost::DataType::kUInt32
@ kUInt32
xgboost::MetaInfo::Extend
void Extend(MetaInfo const &that, bool accumulate_rows)
xgboost::bst_ulong
uint64_t bst_ulong
unsigned long integers
Definition: base.h:117
xgboost::BatchParam::gpu_page_size
size_t gpu_page_size
Page size for external memory mode.
Definition: data.h:215
xgboost::HostSparsePageView::offset
common::Span< bst_row_t const > offset
Definition: data.h:228
xgboost::bst_feature_t
uint32_t bst_feature_t
Type for data column (feature) index.
Definition: base.h:123
xgboost::DMatrix::GetSortedColumnBatches
virtual BatchSet< SortedCSCPage > GetSortedColumnBatches()=0
xgboost::HostSparsePageView::operator[]
Inst operator[](size_t i) const
Definition: data.h:231
xgboost::bst_omp_uint
dmlc::omp_uint bst_omp_uint
define unsigned int for openmp loop
Definition: base.h:270
xgboost::DMatrix::DMatrix
DMatrix()=default
default constructor
xgboost::HostDeviceVector::HostVector
std::vector< T > & HostVector()
DataIterResetCallback
XGB_EXTERN_C typedef void DataIterResetCallback(DataIterHandle handle)
Callback function prototype for reseting external iterator.
xgboost::BatchParam::gpu_id
int gpu_id
The GPU device to use.
Definition: data.h:211
xgboost::MetaInfo::feature_types
HostDeviceVector< FeatureType > feature_types
Definition: data.h:91
span.h
xgboost::DMatrix::EllpackExists
virtual bool EllpackExists() const =0
xgboost::Entry::Entry
XGBOOST_DEVICE Entry(bst_feature_t index, bst_float fvalue)
constructor with index and value
Definition: data.h:196
xgboost::EllpackPage::~EllpackPage
~EllpackPage()
Destructor.
xgboost::DMatrix::IsDense
bool IsDense() const
Whether the matrix is dense.
Definition: data.h:488
xgboost::SortedCSCPage
Definition: data.h:344
xgboost::Entry::operator==
bool operator==(const Entry &other) const
Definition: data.h:201
xgboost::BatchIterator::operator!=
bool operator!=(const BatchIterator &) const
Definition: data.h:424
dmlc::serializer::Handler< xgboost::Entry >::Write
static void Write(Stream *strm, const xgboost::Entry &data)
Definition: data.h:604
xgboost::MetaInfo::GetWeight
bst_float GetWeight(size_t i) const
Get weight of each instances.
Definition: data.h:115
xgboost::HostSparsePageView::Size
size_t Size() const
Definition: data.h:237
xgboost::SparsePage::GetView
HostSparsePageView GetView() const
Definition: data.h:264
xgboost::BatchParam
Parameters for constructing batches.
Definition: data.h:209
xgboost::SparsePage::base_rowid
size_t base_rowid
Definition: data.h:250
xgboost::HostSparsePageView
Definition: data.h:225
xgboost::DataType
DataType
data type accepted by xgboost interface
Definition: data.h:29
xgboost::DMatrix::SparsePageExists
virtual bool SparsePageExists() const =0
xgboost::DMatrix::Create
static DMatrix * Create(AdapterT *adapter, float missing, int nthread, const std::string &cache_prefix="", size_t page_size=kPageSize)
Creates a new DMatrix from an external data adapter.
xgboost::FeatureType::kNumerical
@ kNumerical
xgboost::DMatrix::kPageSize
static const size_t kPageSize
Number of rows per page in external memory. Approximately 100MB per page for dataset with 100 feature...
Definition: data.h:554
xgboost::SparsePage::MemCostBytes
size_t MemCostBytes() const
Definition: data.h:280
XGBOOST_PARALLEL_SORT
#define XGBOOST_PARALLEL_SORT(X, Y, Z)
Definition: base.h:68
xgboost::SparsePage::PushCSC
void PushCSC(const SparsePage &batch)
Push a SparsePage stored in CSC format.
xgboost::MetaInfo::Validate
void Validate(int32_t device) const
Validate all metainfo.
xgboost::SparsePage::GetTranspose
SparsePage GetTranspose(int num_columns) const
xgboost::EllpackPage::Impl
EllpackPageImpl * Impl()
Definition: data.h:387
xgboost::MetaInfo::feature_weigths
HostDeviceVector< float > feature_weigths
Definition: data.h:96
xgboost::HostDeviceVector::Size
size_t Size() const
xgboost::BatchIteratorImpl::operator*
virtual T & operator*()=0
xgboost::FeatureType
FeatureType
Definition: data.h:37
xgboost::CSCPage
Definition: data.h:338
xgboost::Entry::Entry
Entry()=default
default constructor
xgboost::BatchIteratorImpl::AtEnd
virtual bool AtEnd() const =0
xgboost::BatchSet::end
BatchIterator< T > end()
Definition: data.h:443
xgboost::DataType::kStr
@ kStr
xgboost::DMatrix::GetColumnBatches
virtual BatchSet< CSCPage > GetColumnBatches()=0
xgboost::MetaInfo::weights_
HostDeviceVector< bst_float > weights_
weights of each instance, optional
Definition: data.h:64
xgboost::EllpackPage::EllpackPage
EllpackPage()
Default constructor.
xgboost::MetaInfo::num_nonzero_
uint64_t num_nonzero_
number of nonzero entries in the data
Definition: data.h:55
xgboost::XGBAPIThreadLocalEntry
entry to to easily hold returning information
Definition: learner.h:34
xgboost::DataType::kFloat32
@ kFloat32
xgboost::DMatrix::Slice
virtual DMatrix * Slice(common::Span< int32_t const > ridxs)=0
xgboost::common::Span
span class implementation, based on ISO++20 span<T>. The interface should be the same.
Definition: span.h:137
xgboost::DMatrix::Load
static DMatrix * Load(const std::string &uri, bool silent, bool load_row_split, const std::string &file_format="auto", size_t page_size=kPageSize)
Load DMatrix from URI.
std
Definition: intrusive_ptr.h:206
xgboost::BatchParam::BatchParam
BatchParam(int32_t device, int32_t max_bin, size_t gpu_page_size=0)
Definition: data.h:217
xgboost::MetaInfo::operator=
MetaInfo & operator=(MetaInfo &&that)=default
xgboost::MetaInfo::SaveBinary
void SaveBinary(dmlc::Stream *fo) const
Save the Meta info to binary stream.
xgboost::SparsePage::offset
HostDeviceVector< bst_row_t > offset
Definition: data.h:246
xgboost::MetaInfo::GetInfo
void GetInfo(char const *key, bst_ulong *out_len, DataType dtype, const void **out_dptr) const
xgboost::SparsePage::operator[]
Inst operator[](size_t i) const
get i-th row from the batch
Definition: data.h:256
DMatrixHandle
void * DMatrixHandle
handle to DMatrix
Definition: c_api.h:30
xgboost::common::Span::size
constexpr XGBOOST_DEVICE index_type size() const __span_noexcept
Definition: span.h:542
xgboost::BatchIterator
Definition: data.h:404
xgboost::common::Span::data
constexpr XGBOOST_DEVICE pointer data() const __span_noexcept
Definition: span.h:537
xgboost::Entry::CmpValue
static bool CmpValue(const Entry &a, const Entry &b)
reversely compare feature values
Definition: data.h:198
xgboost::MetaInfo
Meta information about dataset, always sit in memory.
Definition: data.h:45
xgboost::DMatrix::GetThreadLocal
XGBAPIThreadLocalEntry & GetThreadLocal() const
Get thread local memory for returning data from DMatrix.
xgboost::EllpackPage::SetBaseRowId
void SetBaseRowId(size_t row_id)
Set the base row id for this page.
dmlc::DMLC_DECLARE_TRAITS
DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true)
xgboost::Entry::fvalue
bst_float fvalue
feature value
Definition: data.h:188
xgboost::DMatrix::GetBatches
BatchSet< T > GetBatches(const BatchParam &param={})
Gets batches. Use range based for loop over BatchSet to access individual batches.
xgboost::MetaInfo::labels_
HostDeviceVector< bst_float > labels_
label of each instance
Definition: data.h:57
xgboost::FeatureType::kCategorical
@ kCategorical
XGBOOST_DEVICE
#define XGBOOST_DEVICE
Tag function as usable by device.
Definition: base.h:84
xgboost::DMatrix::SetInfo
virtual void SetInfo(const char *key, const void *dptr, DataType dtype, size_t num)
Definition: data.h:460
xgboost::DMatrix::PageExists
bool PageExists() const
xgboost::SparsePage::data
HostDeviceVector< Entry > data
the data of the segments
Definition: data.h:248
dmlc::serializer::Handler< xgboost::Entry >::Read
static bool Read(Stream *strm, xgboost::Entry *data)
Definition: data.h:609
xgboost::BatchIterator::iterator_category
std::forward_iterator_tag iterator_category
Definition: data.h:406
xgboost::MetaInfo::kNumField
static constexpr uint64_t kNumField
number of data fields in MetaInfo
Definition: data.h:48
xgboost::BatchSet
Definition: data.h:439
xgboost
namespace of xgboost
Definition: base.h:110
xgboost::SparsePage::Clear
void Clear()
clear the page
Definition: data.h:285
xgboost::MetaInfo::labels_upper_bound_
HostDeviceVector< bst_float > labels_upper_bound_
upper bound of the label, to be used for survival analysis (censored regression)
Definition: data.h:78
xgboost::bst_float
float bst_float
float type, used for storing statistics
Definition: base.h:119
xgboost::SparsePage::SparsePage
SparsePage()
constructor
Definition: data.h:270