Go to the documentation of this file.
7 #ifndef XGBOOST_DATA_H_
8 #define XGBOOST_DATA_H_
10 #include <dmlc/base.h>
11 #include <dmlc/data.h>
12 #include <dmlc/serializer.h>
107 void Validate(int32_t device)
const;
121 return label_order_cache_;
124 std::iota(label_order_cache_.begin(), label_order_cache_.end(), 0);
127 [&l](
size_t i1,
size_t i2) {return std::abs(l[i1]) < std::abs(l[i2]);});
129 return label_order_cache_;
150 void SetInfo(
const char* key,
const void* dptr,
DataType dtype,
size_t num);
160 void SetInfo(
const char* key, std::string
const& interface_str);
163 const void** out_dptr)
const;
166 void GetFeatureInfo(
const char *field, std::vector<std::string>* out_str_vecs)
const;
180 mutable std::vector<size_t> label_order_cache_;
202 return (this->index == other.
index && this->fvalue == other.
fvalue);
257 const auto& data_vec =
data.HostVector();
259 size_t size = offset_vec[i + 1] - offset_vec[i];
260 return {data_vec.data() + offset_vec[i],
289 offset_vec.push_back(0);
290 data.HostVector().clear();
302 #pragma omp parallel for default(none) shared(ncol) schedule(dynamic, 1)
306 this->data.
HostVector().begin() + this->offset.HostVector()[i],
307 this->data.HostVector().begin() + this->offset.HostVector()[i + 1],
323 template <
typename AdapterBatchT>
324 uint64_t
Push(
const AdapterBatchT& batch,
float missing,
int nthread);
350 class EllpackPageImpl;
386 const EllpackPageImpl*
Impl()
const {
return impl_.get(); }
387 EllpackPageImpl*
Impl() {
return impl_.get(); }
390 std::unique_ptr<EllpackPageImpl> impl_;
400 virtual bool AtEnd()
const = 0;
410 CHECK(impl_ !=
nullptr);
415 CHECK(impl_ !=
nullptr);
420 CHECK(impl_ !=
nullptr);
425 CHECK(impl_ !=
nullptr);
426 return !impl_->AtEnd();
430 CHECK(impl_ !=
nullptr);
431 return impl_->AtEnd();
435 std::shared_ptr<BatchIteratorImpl<T>> impl_;
449 struct XGBAPIThreadLocalEntry;
464 virtual void SetInfo(
const char* key, std::string
const& interface_str) {
478 template <
typename T>
505 const std::string& file_format =
"auto",
520 template <
typename AdapterT>
521 static DMatrix*
Create(AdapterT* adapter,
float missing,
int nthread,
522 const std::string& cache_prefix =
"",
572 inline bool DMatrix::PageExists<EllpackPage>()
const {
577 inline bool DMatrix::PageExists<SparsePage>()
const {
600 namespace serializer {
605 strm->Write(data.
index);
610 return strm->Read(&data->
index) && strm->Read(&data->
fvalue);
616 #endif // XGBOOST_DATA_H_
virtual void operator++()=0
bst_feature_t index
feature index
Definition: data.h:186
virtual MetaInfo & Info()=0
meta information of the dataset
int max_bin
Maximum number of bins per feature for histograms.
Definition: data.h:213
virtual BatchSet< EllpackPage > GetEllpackBatches(const BatchParam ¶m)=0
common::Span< Entry const > data
Definition: data.h:229
virtual ~BatchIteratorImpl()=default
size_t Size() const
Definition: data.h:275
void SetBaseRowId(size_t row_id)
Set the base row id for this page.
Definition: data.h:294
CSCPage()
Definition: data.h:340
virtual ~DMatrix()
virtual destructor
A page stored in ELLPACK format.
Definition: data.h:357
In-memory storage unit of sparse batch, stored in CSR format.
Definition: data.h:243
uint64_t Push(const AdapterBatchT &batch, float missing, int nthread)
Pushes external data batch onto this page.
CSCPage(SparsePage page)
Definition: data.h:341
std::size_t index_type
Definition: span.h:416
void SortRows()
Definition: data.h:300
Element from a sparse vector.
Definition: data.h:184
void operator++()
Definition: data.h:409
const EllpackPageImpl * Impl() const
Definition: data.h:386
virtual BatchSet< SparsePage > GetRowBatches()=0
BatchSet(BatchIterator< T > begin_iter)
Definition: data.h:441
A device-and-host vector abstraction layer.
virtual bool SingleColBlock() const =0
SortedCSCPage(SparsePage page)
Definition: data.h:347
bool AtEnd() const
Definition: data.h:429
SortedCSCPage()
Definition: data.h:346
virtual void SetInfo(const char *key, std::string const &interface_str)
Definition: data.h:464
BatchIterator(BatchIteratorImpl< T > *impl)
Definition: data.h:407
defines configuration macros of xgboost.
bool operator!=(const BatchParam &other) const
Definition: data.h:219
common::Span< T const > ConstHostSpan() const
Definition: host_device_vector.h:114
void * DataIterHandle
handle to a external data iterator
Definition: c_api.h:191
BatchIterator< T > begin()
Definition: data.h:442
const T & operator*() const
Definition: data.h:419
T & operator*()
Definition: data.h:414
Internal data structured used by XGBoost during training.
Definition: data.h:454
XGB_EXTERN_C typedef int XGDMatrixCallbackNext(DataIterHandle iter)
Callback function prototype for getting next batch of data.
uint64_t bst_ulong
unsigned long integers
Definition: base.h:117
size_t gpu_page_size
Page size for external memory mode.
Definition: data.h:215
common::Span< bst_row_t const > offset
Definition: data.h:228
uint32_t bst_feature_t
Type for data column (feature) index.
Definition: base.h:123
virtual BatchSet< SortedCSCPage > GetSortedColumnBatches()=0
Inst operator[](size_t i) const
Definition: data.h:231
dmlc::omp_uint bst_omp_uint
define unsigned int for openmp loop
Definition: base.h:270
DMatrix()=default
default constructor
std::vector< T > & HostVector()
XGB_EXTERN_C typedef void DataIterResetCallback(DataIterHandle handle)
Callback function prototype for reseting external iterator.
int gpu_id
The GPU device to use.
Definition: data.h:211
virtual bool EllpackExists() const =0
XGBOOST_DEVICE Entry(bst_feature_t index, bst_float fvalue)
constructor with index and value
Definition: data.h:196
~EllpackPage()
Destructor.
bool IsDense() const
Whether the matrix is dense.
Definition: data.h:488
bool operator==(const Entry &other) const
Definition: data.h:201
bool operator!=(const BatchIterator &) const
Definition: data.h:424
static void Write(Stream *strm, const xgboost::Entry &data)
Definition: data.h:604
size_t Size() const
Definition: data.h:237
HostSparsePageView GetView() const
Definition: data.h:264
Parameters for constructing batches.
Definition: data.h:209
size_t base_rowid
Definition: data.h:250
DataType
data type accepted by xgboost interface
Definition: data.h:29
virtual bool SparsePageExists() const =0
static DMatrix * Create(AdapterT *adapter, float missing, int nthread, const std::string &cache_prefix="", size_t page_size=kPageSize)
Creates a new DMatrix from an external data adapter.
static const size_t kPageSize
Number of rows per page in external memory. Approximately 100MB per page for dataset with 100 feature...
Definition: data.h:554
size_t MemCostBytes() const
Definition: data.h:280
#define XGBOOST_PARALLEL_SORT(X, Y, Z)
Definition: base.h:68
void PushCSC(const SparsePage &batch)
Push a SparsePage stored in CSC format.
SparsePage GetTranspose(int num_columns) const
EllpackPageImpl * Impl()
Definition: data.h:387
virtual T & operator*()=0
FeatureType
Definition: data.h:37
Entry()=default
default constructor
virtual bool AtEnd() const =0
BatchIterator< T > end()
Definition: data.h:443
virtual BatchSet< CSCPage > GetColumnBatches()=0
EllpackPage()
Default constructor.
entry to to easily hold returning information
Definition: learner.h:34
virtual DMatrix * Slice(common::Span< int32_t const > ridxs)=0
span class implementation, based on ISO++20 span<T>. The interface should be the same.
Definition: span.h:137
static DMatrix * Load(const std::string &uri, bool silent, bool load_row_split, const std::string &file_format="auto", size_t page_size=kPageSize)
Load DMatrix from URI.
Definition: intrusive_ptr.h:206
BatchParam(int32_t device, int32_t max_bin, size_t gpu_page_size=0)
Definition: data.h:217
HostDeviceVector< bst_row_t > offset
Definition: data.h:246
Inst operator[](size_t i) const
get i-th row from the batch
Definition: data.h:256
void * DMatrixHandle
handle to DMatrix
Definition: c_api.h:30
constexpr XGBOOST_DEVICE index_type size() const __span_noexcept
Definition: span.h:542
constexpr XGBOOST_DEVICE pointer data() const __span_noexcept
Definition: span.h:537
static bool CmpValue(const Entry &a, const Entry &b)
reversely compare feature values
Definition: data.h:198
XGBAPIThreadLocalEntry & GetThreadLocal() const
Get thread local memory for returning data from DMatrix.
void SetBaseRowId(size_t row_id)
Set the base row id for this page.
DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true)
bst_float fvalue
feature value
Definition: data.h:188
BatchSet< T > GetBatches(const BatchParam ¶m={})
Gets batches. Use range based for loop over BatchSet to access individual batches.
#define XGBOOST_DEVICE
Tag function as usable by device.
Definition: base.h:84
virtual void SetInfo(const char *key, const void *dptr, DataType dtype, size_t num)
Definition: data.h:460
HostDeviceVector< Entry > data
the data of the segments
Definition: data.h:248
static bool Read(Stream *strm, xgboost::Entry *data)
Definition: data.h:609
std::forward_iterator_tag iterator_category
Definition: data.h:406
namespace of xgboost
Definition: base.h:110
void Clear()
clear the page
Definition: data.h:285
float bst_float
float type, used for storing statistics
Definition: base.h:119
SparsePage()
constructor
Definition: data.h:270