Skip to content

Commit bf814bb

Browse files
romainfrancoisromainfrancois
committed
Using arrow::internal::BitmapReader
1 parent c8aa703 commit bf814bb

File tree

4 files changed

+370
-365
lines changed

4 files changed

+370
-365
lines changed

‎r/R/RcppExports.R‎

Lines changed: 80 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,70 @@
11
# Generated by using Rcpp::compileAttributes() -> do not edit by hand
22
# Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393
33

4+
Array__from_vector <- function(x) {
5+
.Call(`_arrow_Array__from_vector`, x)
6+
}
7+
8+
Array__as_vector <- function(array) {
9+
.Call(`_arrow_Array__as_vector`, array)
10+
}
11+
12+
Array__Slice1 <- function(array, offset) {
13+
.Call(`_arrow_Array__Slice1`, array, offset)
14+
}
15+
16+
Array__Slice2 <- function(array, offset, length) {
17+
.Call(`_arrow_Array__Slice2`, array, offset, length)
18+
}
19+
20+
Array__IsNull <- function(x, i) {
21+
.Call(`_arrow_Array__IsNull`, x, i)
22+
}
23+
24+
Array__IsValid <- function(x, i) {
25+
.Call(`_arrow_Array__IsValid`, x, i)
26+
}
27+
28+
Array__length <- function(x) {
29+
.Call(`_arrow_Array__length`, x)
30+
}
31+
32+
Array__offset <- function(x) {
33+
.Call(`_arrow_Array__offset`, x)
34+
}
35+
36+
Array__null_count <- function(x) {
37+
.Call(`_arrow_Array__null_count`, x)
38+
}
39+
40+
Array__type <- function(x) {
41+
.Call(`_arrow_Array__type`, x)
42+
}
43+
44+
Array__ToString <- function(x) {
45+
.Call(`_arrow_Array__ToString`, x)
46+
}
47+
48+
Array__type_id <- function(x) {
49+
.Call(`_arrow_Array__type_id`, x)
50+
}
51+
52+
Array__Equals <- function(lhs, rhs) {
53+
.Call(`_arrow_Array__Equals`, lhs, rhs)
54+
}
55+
56+
Array__ApproxEquals <- function(lhs, rhs) {
57+
.Call(`_arrow_Array__ApproxEquals`, lhs, rhs)
58+
}
59+
60+
Array__data <- function(array) {
61+
.Call(`_arrow_Array__data`, array)
62+
}
63+
64+
Array__RangeEquals <- function(self, other, start_idx, end_idx, other_start_idx) {
65+
.Call(`_arrow_Array__RangeEquals`, self, other, start_idx, end_idx, other_start_idx)
66+
}
67+
468
ArrayData__get_type <- function(x) {
569
.Call(`_arrow_ArrayData__get_type`, x)
670
}
@@ -237,6 +301,22 @@ Object__pointer_address <- function(obj) {
237301
.Call(`_arrow_Object__pointer_address`, obj)
238302
}
239303

304+
Field__initialize <- function(name, type, nullable = TRUE) {
305+
.Call(`_arrow_Field__initialize`, name, type, nullable)
306+
}
307+
308+
Field__ToString <- function(type) {
309+
.Call(`_arrow_Field__ToString`, type)
310+
}
311+
312+
Field__name <- function(type) {
313+
.Call(`_arrow_Field__name`, type)
314+
}
315+
316+
Field__nullable <- function(type) {
317+
.Call(`_arrow_Field__nullable`, type)
318+
}
319+
240320
MemoryPool_default <- function() {
241321
.Call(`_arrow_MemoryPool_default`)
242322
}
@@ -337,83 +417,3 @@ Table__column <- function(table, i) {
337417
.Call(`_arrow_Table__column`, table, i)
338418
}
339419

340-
Array__from_vector <- function(x) {
341-
.Call(`_arrow_Array__from_vector`, x)
342-
}
343-
344-
Array__as_vector <- function(array) {
345-
.Call(`_arrow_Array__as_vector`, array)
346-
}
347-
348-
Array__Slice1 <- function(array, offset) {
349-
.Call(`_arrow_Array__Slice1`, array, offset)
350-
}
351-
352-
Array__Slice2 <- function(array, offset, length) {
353-
.Call(`_arrow_Array__Slice2`, array, offset, length)
354-
}
355-
356-
Array__IsNull <- function(x, i) {
357-
.Call(`_arrow_Array__IsNull`, x, i)
358-
}
359-
360-
Array__IsValid <- function(x, i) {
361-
.Call(`_arrow_Array__IsValid`, x, i)
362-
}
363-
364-
Array__length <- function(x) {
365-
.Call(`_arrow_Array__length`, x)
366-
}
367-
368-
Array__offset <- function(x) {
369-
.Call(`_arrow_Array__offset`, x)
370-
}
371-
372-
Array__null_count <- function(x) {
373-
.Call(`_arrow_Array__null_count`, x)
374-
}
375-
376-
Array__type <- function(x) {
377-
.Call(`_arrow_Array__type`, x)
378-
}
379-
380-
Array__ToString <- function(x) {
381-
.Call(`_arrow_Array__ToString`, x)
382-
}
383-
384-
Array__type_id <- function(x) {
385-
.Call(`_arrow_Array__type_id`, x)
386-
}
387-
388-
Array__Equals <- function(lhs, rhs) {
389-
.Call(`_arrow_Array__Equals`, lhs, rhs)
390-
}
391-
392-
Array__ApproxEquals <- function(lhs, rhs) {
393-
.Call(`_arrow_Array__ApproxEquals`, lhs, rhs)
394-
}
395-
396-
Array__data <- function(array) {
397-
.Call(`_arrow_Array__data`, array)
398-
}
399-
400-
Array__RangeEquals <- function(self, other, start_idx, end_idx, other_start_idx) {
401-
.Call(`_arrow_Array__RangeEquals`, self, other, start_idx, end_idx, other_start_idx)
402-
}
403-
404-
Field__initialize <- function(name, type, nullable = TRUE) {
405-
.Call(`_arrow_Field__initialize`, name, type, nullable)
406-
}
407-
408-
Field__ToString <- function(type) {
409-
.Call(`_arrow_Field__ToString`, type)
410-
}
411-
412-
Field__name <- function(type) {
413-
.Call(`_arrow_Field__name`, type)
414-
}
415-
416-
Field__nullable <- function(type) {
417-
.Call(`_arrow_Field__nullable`, type)
418-
}
419-

‎r/src/ChunkedArray.cpp‎

Lines changed: 32 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -21,39 +21,42 @@ using namespace Rcpp;
2121
using namespace arrow;
2222

2323
template <int RTYPE>
24-
inline SEXP simple_ChunkedArray_to_Vector(const std::shared_ptr<arrow::ChunkedArray>& chunked_array){
25-
using stored_type = typename Rcpp::Vector<RTYPE>::stored_type;
26-
Rcpp::Vector<RTYPE> out = no_init(chunked_array->length());
27-
auto p = out.begin();
28-
29-
int k = 0;
30-
for (int i=0; i<chunked_array->num_chunks(); i++) {
31-
auto chunk = chunked_array->chunk(i);
32-
auto n = chunk->length();
33-
34-
// copy the data
35-
auto q = p;
36-
p = std::copy_n(
37-
reinterpret_cast<const stored_type*>(
38-
chunk->data()->buffers[1]->data() + chunk->offset() * sizeof(stored_type)
39-
),
40-
n, p);
41-
42-
// set NA using the bitmap, TODO
43-
auto bitmap_data = chunk->null_bitmap();
44-
if (bitmap_data && RTYPE != RAWSXP) {
45-
auto data = bitmap_data->data();
46-
for (int j=0; j<n; j++){
47-
if (BitUtil::GetBit(data, j)) {
48-
q[k+j] = Rcpp::Vector<RTYPE>::get_na();
49-
}
24+
inline SEXP simple_ChunkedArray_to_Vector(const std::shared_ptr<arrow::ChunkedArray>& chunked_array){
25+
using stored_type = typename Rcpp::Vector<RTYPE>::stored_type;
26+
Rcpp::Vector<RTYPE> out = no_init(chunked_array->length());
27+
auto p = out.begin();
28+
29+
int k = 0;
30+
for (int i=0; i<chunked_array->num_chunks(); i++) {
31+
auto chunk = chunked_array->chunk(i);
32+
auto n = chunk->length();
33+
34+
// copy the data
35+
auto q = p;
36+
p = std::copy_n(
37+
reinterpret_cast<const stored_type*>(
38+
chunk->data()->buffers[1]->data() + chunk->offset() * sizeof(stored_type)
39+
),
40+
n, p);
41+
42+
// set NA using the bitmap, TODO
43+
auto bitmap_data = chunk->null_bitmap();
44+
if (bitmap_data && RTYPE != RAWSXP) {
45+
arrow::internal::BitmapReader bitmap_reader(
46+
bitmap_data->data(), chunk->offset(), n
47+
);
48+
49+
for (int j=0; j<n; j++, bitmap_reader.Next()){
50+
if (bitmap_reader.IsSet()) {
51+
q[k+j] = Rcpp::Vector<RTYPE>::get_na();
5052
}
5153
}
52-
53-
k += chunk->length();
5454
}
55-
return out;
55+
56+
k += chunk->length();
5657
}
58+
return out;
59+
}
5760

5861

5962
// [[Rcpp::export]]

0 commit comments

Comments
 (0)