7 #ifndef XGBOOST_DATA_H_
8 #define XGBOOST_DATA_H_
10 #include <dmlc/base.h>
11 #include <dmlc/data.h>
12 #include <dmlc/serializer.h>
147 const void** out_dptr)
const;
150 void GetFeatureInfo(
const char *field, std::vector<std::string>* out_str_vecs)
const;
207 mutable std::vector<size_t> label_order_cache_;
208 bool has_categorical_{
false};
233 return (this->index == other.
index && this->fvalue == other.
fvalue);
301 p.forbid_regen =
false;
352 [[nodiscard]]
size_t Size()
const {
366 offset_vec.push_back(0);
367 data.HostVector().clear();
388 void Reindex(uint64_t feature_offset, int32_t n_threads);
402 template <
typename AdapterBatchT>
403 uint64_t
Push(
const AdapterBatchT& batch,
float missing,
int nthread);
429 std::shared_ptr<SparsePage const>
page;
440 class GHistIndexMatrix;
449 [[nodiscard]]
virtual bool AtEnd()
const = 0;
450 virtual std::shared_ptr<T const>
Page()
const = 0;
461 CHECK(impl_ !=
nullptr);
467 CHECK(impl_ !=
nullptr);
472 CHECK(impl_ !=
nullptr);
473 return !impl_->AtEnd();
477 CHECK(impl_ !=
nullptr);
478 return impl_->AtEnd();
481 [[nodiscard]] std::shared_ptr<T const>
Page()
const {
482 return impl_->Page();
486 std::shared_ptr<BatchIteratorImpl<T>> impl_;
500 struct XGBAPIThreadLocalEntry;
511 virtual void SetInfo(
const char* key, std::string
const& interface_str) {
512 auto const& ctx = *this->
Ctx();
529 template <
typename T>
531 template <
typename T>
533 template <
typename T>
535 template <
typename T>
572 template <
typename AdapterT>
574 const std::string& cache_prefix =
"",
625 int32_t nthread, std::string cache);
658 inline bool DMatrix::PageExists<EllpackPage>()
const {
663 inline bool DMatrix::PageExists<GHistIndexMatrix>()
const {
668 inline bool DMatrix::PageExists<SparsePage>()
const {
698 inline BatchSet<ExtSparsePage>
DMatrix::GetBatches(Context
const* ctx, BatchParam
const& param) {
708 namespace serializer {
713 strm->Write(data.
index);
718 return strm->Read(&data->
index) && strm->Read(&data->
fvalue);
Defines configuration macros and basic types for xgboost.
#define XGBOOST_DEVICE
Tag function as usable by device.
Definition: base.h:62
virtual BatchIteratorImpl & operator++()=0
std::forward_iterator_tag iterator_category
Definition: data.h:445
virtual std::shared_ptr< T const > Page() const =0
virtual bool AtEnd() const =0
virtual const T & operator*() const =0
virtual ~BatchIteratorImpl()=default
BatchIterator(std::shared_ptr< BatchIteratorImpl< T >> impl)
Definition: data.h:458
std::forward_iterator_tag iterator_category
Definition: data.h:456
BatchIterator(BatchIteratorImpl< T > *impl)
Definition: data.h:457
const T & operator*() const
Definition: data.h:466
std::shared_ptr< T const > Page() const
Definition: data.h:481
BatchIterator & operator++()
Definition: data.h:460
bool operator!=(const BatchIterator &) const
Definition: data.h:471
bool AtEnd() const
Definition: data.h:476
BatchSet(BatchIterator< T > begin_iter)
Definition: data.h:492
BatchIterator< T > begin()
Definition: data.h:493
BatchIterator< T > end()
Definition: data.h:494
CSCPage()
Definition: data.h:419
CSCPage(SparsePage page)
Definition: data.h:420
Internal data structured used by XGBoost during training.
Definition: data.h:505
virtual BatchSet< EllpackPage > GetEllpackBatches(Context const *ctx, BatchParam const ¶m)=0
static DMatrix * Load(const std::string &uri, bool silent=true, DataSplitMode data_split_mode=DataSplitMode::kRow)
Load DMatrix from URI.
virtual BatchSet< SparsePage > GetRowBatches()=0
virtual BatchSet< GHistIndexMatrix > GetGradientIndex(Context const *ctx, BatchParam const ¶m)=0
virtual void SetInfo(const char *key, std::string const &interface_str)
Definition: data.h:511
static DMatrix * Create(DataIterHandle iter, DMatrixHandle proxy, std::shared_ptr< DMatrix > ref, DataIterResetCallback *reset, XGDMatrixCallbackNext *next, float missing, int nthread, bst_bin_t max_bin)
Create a new Quantile based DMatrix used for histogram based algorithm.
virtual BatchSet< ExtSparsePage > GetExtBatches(Context const *ctx, BatchParam const ¶m)=0
BatchSet< T > GetBatches(Context const *ctx)
virtual ~DMatrix()
virtual destructor
virtual MetaInfo & Info()=0
meta information of the dataset
static DMatrix * Create(AdapterT *adapter, float missing, int nthread, const std::string &cache_prefix="", DataSplitMode data_split_mode=DataSplitMode::kRow)
Creates a new DMatrix from an external data adapter.
virtual DMatrix * SliceCol(int num_slices, int slice_id)=0
Slice a DMatrix by columns.
virtual bool GHistIndexExists() const =0
XGBAPIThreadLocalEntry & GetThreadLocal() const
Get thread local memory for returning data from DMatrix.
virtual bool SparsePageExists() const =0
static DMatrix * Create(DataIterHandle iter, DMatrixHandle proxy, DataIterResetCallback *reset, XGDMatrixCallbackNext *next, float missing, int32_t nthread, std::string cache)
Create an external memory DMatrix with callbacks.
virtual DMatrix * Slice(common::Span< int32_t const > ridxs)=0
virtual Context const * Ctx() const =0
Get the context object of this DMatrix. The context is created during construction of DMatrix with us...
virtual bool SingleColBlock() const =0
BatchSet< T > GetBatches()
Gets batches. Use range based for loop over BatchSet to access individual batches.
virtual const MetaInfo & Info() const =0
meta information of the dataset
virtual bool EllpackExists() const =0
virtual BatchSet< CSCPage > GetColumnBatches(Context const *ctx)=0
virtual BatchSet< SortedCSCPage > GetSortedColumnBatches(Context const *ctx)=0
BatchSet< T > GetBatches(Context const *ctx, const BatchParam ¶m)
bool IsDense() const
Whether the matrix is dense.
Definition: data.h:545
DMatrix()=default
default constructor
Sparse page for exporting DMatrix. Same as SparsePage, just a different type to prevent being used in...
Definition: data.h:427
ExtSparsePage(std::shared_ptr< SparsePage const > p)
Definition: data.h:430
std::shared_ptr< SparsePage const > page
Definition: data.h:429
common::Span< T const > ConstHostSpan() const
Definition: host_device_vector.h:115
std::vector< T > & HostVector()
Data structure representing JSON format.
Definition: json.h:368
SortedCSCPage(SparsePage page)
Definition: data.h:436
SortedCSCPage()
Definition: data.h:435
In-memory storage unit of sparse batch, stored in CSR format.
Definition: data.h:324
void Push(const SparsePage &batch)
Push a sparse page.
SparsePage()
constructor
Definition: data.h:341
SparsePage GetTranspose(int num_columns, int32_t n_threads) const
void SetBaseRowId(size_t row_id)
Set the base row id for this page.
Definition: data.h:371
void Reindex(uint64_t feature_offset, int32_t n_threads)
Reindex the column index with an offset.
uint64_t Push(const AdapterBatchT &batch, float missing, int nthread)
Pushes external data batch onto this page.
void PushCSC(const SparsePage &batch)
Push a SparsePage stored in CSC format.
bool IsIndicesSorted(int32_t n_threads) const
Check wether the column index is sorted.
virtual ~SparsePage()=default
void SortIndices(int32_t n_threads)
Sort the column index.
HostDeviceVector< Entry > data
the data of the segments
Definition: data.h:329
HostSparsePageView GetView() const
Definition: data.h:336
SparsePage & operator=(SparsePage const &that)=delete
size_t MemCostBytes() const
Definition: data.h:357
void Clear()
clear the page
Definition: data.h:362
SparsePage(SparsePage const &that)=delete
size_t Size() const
Definition: data.h:352
void SortRows(int32_t n_threads)
HostDeviceVector< bst_idx_t > offset
Definition: data.h:327
SparsePage & operator=(SparsePage &&that)=default
SparsePage(SparsePage &&that)=default
size_t base_rowid
Definition: data.h:331
span class implementation, based on ISO++20 span<T>. The interface should be the same.
Definition: span.h:422
constexpr XGBOOST_DEVICE pointer data() const __span_noexcept
Definition: span.h:547
std::size_t index_type
Definition: span.h:426
constexpr XGBOOST_DEVICE index_type size() const __span_noexcept
Definition: span.h:552
DECLARE_FIELD_ENUM_CLASS(xgboost::DataSplitMode)
void * DMatrixHandle
handle to DMatrix
Definition: c_api.h:50
int XGDMatrixCallbackNext(DataIterHandle iter)
Callback function prototype for getting next batch of data.
Definition: c_api.h:435
void * DataIterHandle
handle to a external data iterator
Definition: c_api.h:348
void DataIterResetCallback(DataIterHandle handle)
Callback function prototype for resetting external iterator.
Definition: c_api.h:440
A device-and-host vector abstraction layer.
Linear algebra related utilities.
DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true)
Definition: intrusive_ptr.h:207
Core data structure for multi-target trees.
Definition: base.h:87
FeatureType
Definition: data.h:40
std::int32_t bst_bin_t
Type for histogram bin index. We sometimes use -1 to indicate invalid bin.
Definition: base.h:101
DataSplitMode
Definition: data.h:42
std::uint32_t bst_feature_t
Type for data column (feature) index.
Definition: base.h:97
DataType
data type accepted by xgboost interface
Definition: data.h:32
std::uint64_t bst_ulong
unsigned long integers
Definition: base.h:91
float bst_float
float type, used for storing statistics
Definition: base.h:93
static void Write(Stream *strm, const xgboost::Entry &data)
Definition: data.h:712
static bool Read(Stream *strm, xgboost::Entry *data)
Definition: data.h:717
Parameters for constructing histogram index batches.
Definition: data.h:240
bool forbid_regen
Forbid regenerating the gradient index. Used for internal validation.
Definition: data.h:257
bst_bin_t max_bin
Maximum number of bins per feature for histograms.
Definition: data.h:244
common::Span< float const > hess
Hessian, used for sketching with future approx implementation.
Definition: data.h:248
bool regen
Whether should we force DMatrix to regenerate the batch. Only used for GHistIndex.
Definition: data.h:253
bool ParamNotEqual(BatchParam const &other) const
Definition: data.h:281
BatchParam()=default
Exact or others that don't need histogram.
double sparse_thresh
Parameter used to generate column matrix for hist.
Definition: data.h:261
bool Initialized() const
Definition: data.h:292
BatchParam(bst_bin_t max_bin, common::Span< float const > hessian, bool regenerate)
Used by the approx tree method.
Definition: data.h:278
BatchParam MakeCache() const
Make a copy of self for DMatrix to describe how its existing index was generated.
Definition: data.h:296
BatchParam(bst_bin_t max_bin, double sparse_thresh)
Used by the hist tree method.
Definition: data.h:270
Runtime context for XGBoost. Contains information like threads and device.
Definition: context.h:133
A type for device ordinal. The type is packed into 32-bit for efficient use in viewing types like lin...
Definition: context.h:34
Element from a sparse vector.
Definition: data.h:212
XGBOOST_DEVICE Entry(bst_feature_t index, bst_float fvalue)
constructor with index and value
Definition: data.h:224
Entry()=default
default constructor
bst_feature_t index
feature index
Definition: data.h:214
static bool CmpIndex(Entry const &a, Entry const &b)
Definition: data.h:229
bst_float fvalue
feature value
Definition: data.h:216
bool operator==(const Entry &other) const
Definition: data.h:232
static bool CmpValue(const Entry &a, const Entry &b)
reversely compare feature values
Definition: data.h:226
size_t Size() const
Definition: data.h:318
common::Span< bst_idx_t const > offset
Definition: data.h:309
common::Span< Entry const > data
Definition: data.h:310
Inst operator[](size_t i) const
Definition: data.h:312
Definition: string_view.h:16