diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt index 1a4902873a2ea..5040366776652 100644 --- a/Framework/Core/CMakeLists.txt +++ b/Framework/Core/CMakeLists.txt @@ -81,6 +81,7 @@ o2_add_library(Framework src/GraphvizHelpers.cxx src/MermaidHelpers.cxx src/HTTPParser.cxx + src/IndexBuilderHelpers.cxx src/InputRecord.cxx src/InputRouteHelpers.cxx src/InputSpan.cxx diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h index 3a86f1c8f1679..b8078e3440e24 100644 --- a/Framework/Core/include/Framework/ASoA.h +++ b/Framework/Core/include/Framework/ASoA.h @@ -1017,6 +1017,10 @@ constexpr bool is_binding_compatible_v() return are_bindings_compatible_v(originals_pack_t{}); } +template +struct is_binding_compatible : std::conditional_t(), std::true_type, std::false_type> { +}; + //! Helper to check if a type T is an iterator template inline constexpr bool is_soa_iterator_v = framework::is_base_of_template_v || framework::is_specialization_v; @@ -1198,6 +1202,19 @@ class Table { } + template + inline arrow::ChunkedArray* getIndexToKey() + { + if constexpr (framework::has_type_conditional_v) { + using IC = framework::pack_element_t(external_index_columns_t{}), external_index_columns_t>; + return mColumnChunks[framework::has_type_at(persistent_columns_t{})]; + } else if constexpr (std::is_same_v) { + return nullptr; + } else { + static_assert(framework::always_static_assert_v, "This table does not have an index to this type"); + } + } + unfiltered_iterator begin() { return unfiltered_iterator(mBegin); diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h index 8f8d08e047987..36b2bf7386c0f 100644 --- a/Framework/Core/include/Framework/AnalysisHelpers.h +++ b/Framework/Core/include/Framework/AnalysisHelpers.h @@ -20,6 +20,7 @@ #include "Framework/OutputObjHeader.h" #include "Framework/StringHelpers.h" #include "Framework/Output.h" +#include "Framework/IndexBuilderHelpers.h" #include namespace o2::framework { @@ -219,154 +220,95 @@ struct Spawns : TableTransform - static auto indexBuilder(const char* label, framework::pack, Key const&, std::tuple&& tables) - { - static_assert(sizeof...(Cs) == sizeof...(T) + 1, "Number of columns does not coincide with number of supplied tables"); - using tables_t = framework::pack; - using first_t = T1; - auto tail = tuple_tail(tables); - TableBuilder builder; - auto cursor = framework::FFL(builder.cursor>()); - - std::array values; - iterator_tuple_t...> iterators = std::apply( - [](auto&&... x) { - return std::make_tuple(x.begin()...); - }, - tail); - - using rest_it_t = decltype(pack_from_tuple(iterators)); - - auto setValue = [&](auto& x, int idx) -> bool { - using type = std::decay_t; - constexpr auto position = framework::has_type_at_v(rest_it_t{}); - - if constexpr (std::is_same_v...>>, Key>) { - values[position] = idx; - return true; - } else { - lowerBound(idx, x); - if (x == soa::RowViewSentinel{static_cast(x.size())}) { - return false; - } else if (x.template getId() != idx) { - return false; - } else { - values[position] = x.globalIndex(); - ++x; - return true; - } - } - }; +/// Sparse index: values in a row can be (-1), index table is isomorphic (joinable) +/// to T1 +struct Exclusive { +}; +struct Sparse { +}; - auto first = std::get(tables); - for (auto& row : first) { - auto idx = -1; - if constexpr (std::is_same_v) { - idx = row.globalIndex(); - } else { - idx = row.template getId(); - } - if (std::apply( - [](auto&... x) { - return ((x == soa::RowViewSentinel{static_cast(x.size())}) && ...); - }, - iterators)) { - break; - } +namespace +{ +template +inline arrow::ChunkedArray* getIndexToKey(arrow::Table* table) +{ + using IC = framework::pack_element_t(typename T::external_index_columns_t{}), typename T::external_index_columns_t>; + return table->column(framework::has_type_at(typename T::persistent_columns_t{})).get(); +} - auto result = std::apply( - [&](auto&... x) { - std::array results{setValue(x, idx)...}; - return (results[framework::has_type_at_v>(rest_it_t{})] && ...); - }, - iterators); +template +struct ColumnTrait { + static_assert(framework::is_base_of_template_v, "Not a column type!"); + using column_t = C; - if (result) { - cursor(0, row.globalIndex(), values[framework::has_type_at_v(tables_t{})]...); - } + static constexpr auto listSize() + { + if constexpr (std::is_same_v>) { + return -1; + } else if constexpr (std::is_same_v) { + return 2; + } else { + return 1; } - builder.setLabel(label); - return builder.finalize(); } - template - static auto makeIndex(Key const& key, std::tuple&& tables) + template + static std::shared_ptr makeColumnBuilder(arrow::Table* table, arrow::MemoryPool* pool) { - auto t = IDX{indexBuilder(o2::aod::MetadataTrait::metadata::tableLabel(), - typename o2::aod::MetadataTrait::metadata::index_pack_t{}, - key, - std::make_tuple(std::decay_t{{std::get(tables).asArrowTable()}}, std::decay_t{{std::get(tables).asArrowTable()}}...))}; - t.bindExternalIndices(&key, &std::get(tables), &std::get(tables)...); - return t; + if constexpr (!std::is_same_v) { + return std::make_shared(getIndexToKey(table), C::columnLabel(), listSize(), pool); + } else { + return std::make_shared(C::columnLabel(), pool); + } } }; -/// Sparse index: values in a row can be (-1), index table is isomorphic (joinable) -/// to T1 -struct IndexSparse { - template - static auto indexBuilder(const char* label, framework::pack, Key const&, std::tuple&& tables) - { - static_assert(sizeof...(Cs) == sizeof...(T) + 1, "Number of columns does not coincide with number of supplied tables"); - using tables_t = framework::pack; - using first_t = T1; - auto tail = tuple_tail(tables); - TableBuilder builder; - auto cursor = framework::FFL(builder.cursor>()); - - std::array values; - - iterator_tuple_t...> iterators = std::apply( - [](auto&&... x) { - return std::make_tuple(x.begin()...); - }, - tail); - - using rest_it_t = decltype(pack_from_tuple(iterators)); - - auto setValue = [&](auto& x, int idx) -> bool { - using type = std::decay_t; - constexpr auto position = framework::has_type_at_v(rest_it_t{}); - - if constexpr (std::is_same_v...>>, Key>) { - values[position] = idx; - return true; - } else { - lowerBound(idx, x); - if (x == soa::RowViewSentinel{static_cast(x.size())}) { - values[position] = -1; - return false; - } else if (x.template getId() != idx) { - values[position] = -1; - return false; - } else { - values[position] = x.globalIndex(); - ++x; - return true; - } - } - }; - auto first = std::get(tables); - for (auto& row : first) { +template +struct Reduction { + using type = typename std::conditional(), SelfIndexColumnBuilder, IndexColumnBuilder>::type; +}; +} // namespace + +template +struct IndexBuilder { + template + static auto indexBuilder(const char* label, std::vector>&& tables, framework::pack, framework::pack) + { + auto pool = arrow::default_memory_pool(); + SelfIndexColumnBuilder self{C1::columnLabel(), pool}; + std::unique_ptr keyIndex = nullptr; + int64_t counter = 0; + if constexpr (!std::is_same_v) { + keyIndex = std::make_unique(getIndexToKey(tables[0].get())); + } + + std::array, sizeof...(Cs)> columnBuilders{ColumnTrait::template makeColumnBuilder(framework::pack{}), framework::pack>, Key>( + tables[framework::has_type_at_v(framework::pack{}) + 1].get(), + pool)...}; + std::array finds; + + for (counter = 0; counter < tables[0]->num_rows(); ++counter) { auto idx = -1; - if constexpr (std::is_same_v) { - idx = row.globalIndex(); + if constexpr (std::is_same_v) { + idx = counter; } else { - idx = row.template getId(); + idx = keyIndex->valueAt(counter); + } + finds = {std::static_pointer_cast::type>(columnBuilders[framework::has_type_at_v(framework::pack{})])->template find(idx)...}; + if constexpr (std::is_same_v) { + (std::static_pointer_cast::type>(columnBuilders[framework::has_type_at_v(framework::pack{})])->template fill(idx), ...); + self.fill(counter); + } else if constexpr (std::is_same_v) { + if (std::none_of(finds.begin(), finds.end(), [](bool const x) { return x == false; })) { + (std::static_pointer_cast::type>(columnBuilders[framework::has_type_at_v(framework::pack{})])->template fill(idx), ...); + self.fill(counter); + } } - std::apply( - [&](auto&... x) { - (setValue(x, idx), ...); - }, - iterators); - - cursor(0, row.globalIndex(), values[framework::has_type_at_v(tables_t{})]...); } - builder.setLabel(label); - return builder.finalize(); + + return makeArrowTable(label, + {self.template result(), std::static_pointer_cast::type>(columnBuilders[framework::has_type_at_v(framework::pack{})])->template result()...}, + {self.field(), std::static_pointer_cast::type>(columnBuilders[framework::has_type_at_v(framework::pack{})])->field()...}); } template @@ -384,7 +326,7 @@ struct IndexSparse { /// This helper struct allows you to declare index tables to be created in a task template struct Builds : TableTransform::metadata> { - using IP = std::conditional_t::metadata::exclusive, IndexExclusive, IndexSparse>; + using IP = std::conditional_t::metadata::exclusive, IndexBuilder, IndexBuilder>; using Key = typename T::indexing_t; using H = typename T::first_t; using Ts = typename T::rest_t; @@ -410,10 +352,10 @@ struct Builds : TableTransform::metadata> { return index_pack_t{}; } - template - auto build(framework::pack, Key const& key, std::tuple&& tables) + template + auto build(framework::pack, framework::pack, std::vector>&& tables) { - this->table = std::make_shared(IP::indexBuilder(aod::MetadataTrait::metadata::tableLabel(), framework::pack{}, key, std::forward>(tables))); + this->table = std::make_shared(IP::template indexBuilder(aod::MetadataTrait::metadata::tableLabel(), std::forward>>(tables), framework::pack{}, framework::pack{})); return (this->table != nullptr); } }; diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h index 4ea6d86ba8054..4ccc217cd7369 100644 --- a/Framework/Core/include/Framework/AnalysisManagers.h +++ b/Framework/Core/include/Framework/AnalysisManagers.h @@ -346,26 +346,26 @@ struct OutputManager> { }; /// Builds specialization -template -static inline auto extractOriginalsVector(framework::pack, ProcessingContext& pc) +template +static inline auto doExtractOriginal(framework::pack, ProcessingContext& pc) { - return std::vector{extractOriginal(pc)...}; + if constexpr (sizeof...(Ts) == 1) { + return pc.inputs().get(aod::MetadataTrait>>::metadata::tableLabel())->asArrowTable(); + } else { + return std::vector{pc.inputs().get(aod::MetadataTrait::metadata::tableLabel())->asArrowTable()...}; + } } template -static inline auto extractTypedOriginal(ProcessingContext& pc) +static inline auto extractOriginalJoined(ProcessingContext& pc) { - if constexpr (soa::is_type_with_originals_v) { - return O{extractOriginalsVector(soa::originals_pack_t{}, pc)}; - } else { - return O{pc.inputs().get(aod::MetadataTrait::metadata::tableLabel())->asArrowTable()}; - } + return o2::soa::ArrowHelpers::joinTables({doExtractOriginal(soa::make_originals_from_type(), pc)}); } template -static inline auto extractOriginalsTuple(framework::pack, ProcessingContext& pc) +static inline auto extractOriginalsVector(framework::pack, ProcessingContext& pc) { - return std::make_tuple(extractTypedOriginal(pc)...); + return std::vector{extractOriginalJoined(pc)...}; } template @@ -378,9 +378,8 @@ struct OutputManager> { static bool prepare(ProcessingContext& pc, Builds& what) { - return what.build(what.pack(), - extractTypedOriginal::Key>(pc), - extractOriginalsTuple(what.originals_pack(), pc)); + return what.template build(what.pack(), what.originals_pack(), + extractOriginalsVector(what.originals_pack(), pc)); } static bool finalize(ProcessingContext& pc, Builds& what) diff --git a/Framework/Core/include/Framework/IndexBuilderHelpers.h b/Framework/Core/include/Framework/IndexBuilderHelpers.h new file mode 100644 index 0000000000000..7f4a734c74d87 --- /dev/null +++ b/Framework/Core/include/Framework/IndexBuilderHelpers.h @@ -0,0 +1,144 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef O2_FRAMEWORK_INDEXBUILDERHELPERS_H_ +#define O2_FRAMEWORK_INDEXBUILDERHELPERS_H_ +#include "Framework/RuntimeError.h" +#include "arrow/array.h" +#include +#include +#include +#include +#include +#include + +namespace o2::framework +{ +struct ChunkedArrayIterator { + ChunkedArrayIterator(arrow::ChunkedArray* source); + virtual ~ChunkedArrayIterator() = default; + + arrow::ChunkedArray* mSource; + size_t mPosition = 0; + int mChunk = 0; + size_t mOffset = 0; + std::shared_ptr mCurrentArray = nullptr; + int const* mCurrent = nullptr; + int const* mLast = nullptr; + size_t mFirstIndex = 0; + + std::shared_ptr getCurrentArray(); + void nextChunk(); + void prevChunk(); + int valueAt(size_t pos); +}; + +struct SelfIndexColumnBuilder { + SelfIndexColumnBuilder(const char* name, arrow::MemoryPool* pool); + virtual ~SelfIndexColumnBuilder() = default; + + template + inline std::shared_ptr result() const + { + std::shared_ptr array; + auto status = static_cast(mBuilder.get())->Finish(&array); + if (!status.ok()) { + throw runtime_error("Cannot build an array"); + } + + return std::make_shared(array); + } + std::shared_ptr field() const; + template + inline bool find(int) + { + return true; + } + + template + inline void fill(int idx) + { + (void)static_cast(mBuilder.get())->Append(idx); + } + + std::string mColumnName; + std::unique_ptr mBuilder = nullptr; +}; + +class IndexColumnBuilder : public SelfIndexColumnBuilder, public ChunkedArrayIterator +{ + public: + IndexColumnBuilder(arrow::ChunkedArray* source, const char* name, int listSize, arrow::MemoryPool* pool); + ~IndexColumnBuilder() override = default; + + template + inline std::shared_ptr result() const + { + if constexpr (std::is_same_v>) { + return resultMulti(); + } else if constexpr (std::is_same_v) { + return resultSlice(); + } else { + return resultSingle(); + } + } + + template + inline bool find(int idx) + { + if constexpr (std::is_same_v>) { + return findMulti(idx); + } else if constexpr (std::is_same_v) { + return findSlice(idx); + } else { + return findSingle(idx); + } + } + + template + inline void fill(int idx) + { + ++mResultSize; + if constexpr (std::is_same_v>) { + fillMulti(idx); + } else if constexpr (std::is_same_v) { + fillSlice(idx); + } else { + fillSingle(idx); + } + } + + private: + bool findSingle(int idx); + bool findSlice(int idx); + bool findMulti(int idx); + + void fillSingle(int idx); + void fillSlice(int idx); + void fillMulti(int idx); + + std::shared_ptr resultSingle() const; + std::shared_ptr resultSlice() const; + std::shared_ptr resultMulti() const; + + int mListSize = 1; + std::shared_ptr mArrowType; + arrow::ArrayBuilder* mValueBuilder = nullptr; + std::unique_ptr mListBuilder = nullptr; + + size_t mSourceSize = 0; + size_t mResultSize = 0; +}; + +std::shared_ptr makeArrowTable(const char* label, std::vector>&& columns, std::vector>&& fields); +} // namespace o2::framework + +#endif // O2_FRAMEWORK_INDEXBUILDERHELPERS_H_ diff --git a/Framework/Core/src/AODReaderHelpers.cxx b/Framework/Core/src/AODReaderHelpers.cxx index 8899c66009b53..4cccdf7849d15 100644 --- a/Framework/Core/src/AODReaderHelpers.cxx +++ b/Framework/Core/src/AODReaderHelpers.cxx @@ -56,7 +56,7 @@ auto setEOSCallback(InitContext& ic) } template -static inline auto doExtractTypedOriginal(framework::pack, ProcessingContext& pc) +static inline auto doExtractOriginal(framework::pack, ProcessingContext& pc) { if constexpr (sizeof...(Ts) == 1) { return pc.inputs().get(aod::MetadataTrait>>::metadata::tableLabel())->asArrowTable(); @@ -68,7 +68,13 @@ static inline auto doExtractTypedOriginal(framework::pack, ProcessingCont template static inline auto extractTypedOriginal(ProcessingContext& pc) { - return O{doExtractTypedOriginal(soa::make_originals_from_type(), pc)}; + return O{doExtractOriginal(soa::make_originals_from_type(), pc)}; +} + +template +static inline auto extractOriginal(ProcessingContext& pc) +{ + return o2::soa::ArrowHelpers::joinTables({doExtractOriginal(soa::make_originals_from_type(), pc)}); } template @@ -77,6 +83,12 @@ static inline auto extractOriginalsTuple(framework::pack, ProcessingConte return std::make_tuple(extractTypedOriginal(pc)...); } +template +static inline auto extractOriginalsVector(framework::pack, ProcessingContext& pc) +{ + return std::vector{extractOriginal(pc)...}; +} + AlgorithmSpec AODReaderHelpers::indexBuilderCallback(std::vector& requested) { return AlgorithmSpec::InitCallback{[requested](InitContext& ic) { @@ -91,13 +103,15 @@ AlgorithmSpec AODReaderHelpers::indexBuilderCallback(std::vector& req using index_pack_t = typename metadata_t::index_pack_t; using originals = typename metadata_t::originals; if constexpr (metadata_t::exclusive == true) { - return o2::framework::IndexExclusive::indexBuilder(input.binding.c_str(), index_pack_t{}, - extractTypedOriginal(pc), - extractOriginalsTuple(originals{}, pc)); + return o2::framework::IndexBuilder::indexBuilder(input.binding.c_str(), + extractOriginalsVector(originals{}, pc), + index_pack_t{}, + originals{}); } else { - return o2::framework::IndexSparse::indexBuilder(input.binding.c_str(), index_pack_t{}, - extractTypedOriginal(pc), - extractOriginalsTuple(originals{}, pc)); + return o2::framework::IndexBuilder::indexBuilder(input.binding.c_str(), + extractOriginalsVector(originals{}, pc), + index_pack_t{}, + originals{}); } }; diff --git a/Framework/Core/src/IndexBuilderHelpers.cxx b/Framework/Core/src/IndexBuilderHelpers.cxx new file mode 100644 index 0000000000000..e9abe1a40210c --- /dev/null +++ b/Framework/Core/src/IndexBuilderHelpers.cxx @@ -0,0 +1,192 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "Framework/IndexBuilderHelpers.h" +#include "Framework/CompilerBuiltins.h" +#include +#include +#include + +namespace o2::framework +{ +ChunkedArrayIterator::ChunkedArrayIterator(arrow::ChunkedArray* source) + : mSource{source} +{ + mCurrentArray = getCurrentArray(); + mCurrent = reinterpret_cast(mCurrentArray->values()->data()) + mOffset; + mLast = mCurrent + mCurrentArray->length(); +} + +SelfIndexColumnBuilder::SelfIndexColumnBuilder(const char* name, arrow::MemoryPool* pool) + : mColumnName{name} +{ + auto status = arrow::MakeBuilder(pool, arrow::int32(), &mBuilder); + if (!status.ok()) { + throw runtime_error("Cannot create array builder!"); + } +} + +std::shared_ptr SelfIndexColumnBuilder::field() const +{ + return std::make_shared(mColumnName, arrow::int32()); +} + +IndexColumnBuilder::IndexColumnBuilder(arrow::ChunkedArray* source, const char* name, int listSize, arrow::MemoryPool* pool) + : ChunkedArrayIterator{source}, + SelfIndexColumnBuilder{name, pool}, + mListSize{listSize}, + mSourceSize{(size_t)source->length()} +{ + switch (mListSize) { + case 1: { + mValueBuilder = mBuilder.get(); + mArrowType = arrow::int32(); + }; break; + case 2: { + mListBuilder = std::make_unique(pool, std::move(mBuilder), mListSize); + mValueBuilder = static_cast(mListBuilder.get())->value_builder(); + mArrowType = arrow::fixed_size_list(arrow::int32(), 2); + }; break; + case -1: { + mListBuilder = std::make_unique(pool, std::move(mBuilder)); + mValueBuilder = static_cast(mListBuilder.get())->value_builder(); + mArrowType = arrow::list(arrow::int32()); + }; break; + default: + throw runtime_error_f("Invalid list size for index column: %d", mListSize); + } +} + +std::shared_ptr IndexColumnBuilder::resultSingle() const +{ + std::shared_ptr array; + auto status = static_cast(mValueBuilder)->Finish(&array); + if (!status.ok()) { + throw runtime_error("Cannot build an array"); + } + return std::make_shared(array); +} + +std::shared_ptr IndexColumnBuilder::resultSlice() const +{ + std::shared_ptr array; + std::shared_ptr varray; + auto status = static_cast(mValueBuilder)->Finish(&varray); + if (!status.ok()) { + throw runtime_error("Cannot build an array"); + } + array = std::make_shared(mArrowType, mResultSize, varray); + return std::make_shared(array); +} + +std::shared_ptr IndexColumnBuilder::resultMulti() const +{ + throw runtime_error("Not implemented"); +} + +bool IndexColumnBuilder::findSingle(int idx) +{ + size_t step = 0; + auto count = mSourceSize - mPosition; + while (count > 0) { + step = count / 2; + mPosition += step; + if (valueAt(mPosition) <= idx) { + count -= step + 1; + } else { + mPosition -= step; + count = step; + } + } + + return (mPosition < mSourceSize && valueAt(mPosition) == idx); +} + +bool IndexColumnBuilder::findSlice(int idx) +{ + throw runtime_error("Not implemented"); +} + +bool IndexColumnBuilder::findMulti(int idx) +{ + throw runtime_error("Not implemented"); +} + +void IndexColumnBuilder::fillSingle(int idx) +{ + // entry point + if (mPosition < mSourceSize && valueAt(mPosition) == idx) { + (void)static_cast(mValueBuilder)->Append((int)mPosition); + } else { + (void)static_cast(mValueBuilder)->Append(-1); + } +} + +void IndexColumnBuilder::fillSlice(int idx) +{ + throw runtime_error("Not implemented"); +} + +void IndexColumnBuilder::fillMulti(int idx) +{ + throw runtime_error("Not implemented"); +} + +std::shared_ptr ChunkedArrayIterator::getCurrentArray() +{ + auto chunk = mSource->chunk(mChunk); + mOffset = chunk->offset(); + return std::static_pointer_cast(chunk); +} + +void ChunkedArrayIterator::nextChunk() +{ + auto previousArray = getCurrentArray(); + mFirstIndex += previousArray->length(); + + ++mChunk; + auto array = getCurrentArray(); + mCurrent = reinterpret_cast(array->values()->data()) + mOffset - mFirstIndex; + mLast = mCurrent + array->length() + mFirstIndex; +} + +void ChunkedArrayIterator::prevChunk() +{ + auto previousArray = getCurrentArray(); + mFirstIndex -= previousArray->length(); + + --mChunk; + auto array = getCurrentArray(); + mCurrent = reinterpret_cast(array->values()->data()) + mOffset - mFirstIndex; + mLast = mCurrent + array->length() + mFirstIndex; +} + +int ChunkedArrayIterator::valueAt(size_t pos) +{ + while (O2_BUILTIN_UNLIKELY(mCurrent + pos >= mLast)) { + nextChunk(); + } + while (O2_BUILTIN_UNLIKELY(pos < mFirstIndex)) { + prevChunk(); + } + return *(mCurrent + pos); +} + +std::shared_ptr makeArrowTable(const char* label, std::vector>&& columns, std::vector>&& fields) +{ + auto schema = std::make_shared(fields); + schema->WithMetadata( + std::make_shared( + std::vector{std::string{"label"}}, + std::vector{std::string{label}})); + return arrow::Table::Make(schema, columns); +} +} // namespace o2::framework diff --git a/Framework/Core/test/test_IndexBuilder.cxx b/Framework/Core/test/test_IndexBuilder.cxx index 388749b514d72..cdc8f91812d8c 100644 --- a/Framework/Core/test/test_IndexBuilder.cxx +++ b/Framework/Core/test/test_IndexBuilder.cxx @@ -102,7 +102,7 @@ BOOST_AUTO_TEST_CASE(TestIndexBuilder) auto t4 = b4.finalize(); Categorys st4{t4}; - auto t5 = IndexExclusive::indexBuilder("test1", typename IDXs::persistent_columns_t{}, st1, std::tie(st1, st2, st3, st4)); + auto t5 = IndexBuilder::indexBuilder("test1a", {t1, t2, t3, t4}, typename IDXs::persistent_columns_t{}, o2::framework::pack{}); BOOST_REQUIRE_EQUAL(t5->num_rows(), 4); IDXs idxt{t5}; idxt.bindExternalIndices(&st1, &st2, &st3, &st4); @@ -112,7 +112,7 @@ BOOST_AUTO_TEST_CASE(TestIndexBuilder) BOOST_REQUIRE(row.category().pointId() == row.pointId()); } - auto t6 = IndexSparse::indexBuilder("test2", typename IDX2s::persistent_columns_t{}, st1, std::tie(st2, st1, st3, st4)); + auto t6 = IndexBuilder::indexBuilder("test3", {t2, t1, t3, t4}, typename IDX2s::persistent_columns_t{}, o2::framework::pack{}); BOOST_REQUIRE_EQUAL(t6->num_rows(), st2.size()); IDX2s idxs{t6}; std::array fs{0, 1, 2, -1, -1, 4, -1}; diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h index b05c309a9eb81..9d415efa85fd3 100644 --- a/Framework/Foundation/include/Framework/Pack.h +++ b/Framework/Foundation/include/Framework/Pack.h @@ -160,6 +160,16 @@ struct has_type> : std::disjunction...> { template inline constexpr bool has_type_v = has_type::value; +template