diff --git a/src/duckdb/extension/core_functions/aggregate/holistic/approx_top_k.cpp b/src/duckdb/extension/core_functions/aggregate/holistic/approx_top_k.cpp index 985e8392c..af8426984 100644 --- a/src/duckdb/extension/core_functions/aggregate/holistic/approx_top_k.cpp +++ b/src/duckdb/extension/core_functions/aggregate/holistic/approx_top_k.cpp @@ -1,5 +1,7 @@ #include "duckdb/common/vector/flat_vector.hpp" +#include "duckdb/common/vector/vector_iterator.hpp" #include "duckdb/common/vector/list_vector.hpp" +#include "duckdb/common/vector/struct_vector.hpp" #include "core_functions/aggregate/histogram_helpers.hpp" #include "core_functions/aggregate/holistic_functions.hpp" #include "duckdb/function/aggregate/sort_key_helpers.hpp" @@ -81,7 +83,7 @@ struct InternalApproxTopKState { filter.resize(filter_size); } - static void CopyValue(ApproxTopKValue &value, const ApproxTopKString &input, AggregateInputData &input_data) { + static void CopyValue(ApproxTopKValue &value, const ApproxTopKString &input, ArenaAllocator &allocator) { value.str_val.hash = input.hash; if (input.str.IsInlined()) { // no need to copy @@ -92,7 +94,7 @@ struct InternalApproxTopKState { if (value.size > value.capacity) { // need to re-allocate for this value value.capacity = UnsafeNumericCast(NextPowerOfTwo(value.size)); - value.dataptr = char_ptr_cast(input_data.allocator.Allocate(value.capacity)); + value.dataptr = char_ptr_cast(allocator.Allocate(value.capacity)); } // copy over the data memcpy(value.dataptr, input.str.GetData(), value.size); @@ -129,7 +131,7 @@ struct InternalApproxTopKState { filter[value.str_val.hash & filter_mask] = value.count; lookup_map.erase(value.str_val); } - CopyValue(value, input, aggr_input); + CopyValue(value, input, aggr_input.allocator); lookup_map.insert(make_pair(value.str_val, reference(value))); IncrementCount(value, increment); } @@ -378,6 +380,168 @@ void ApproxTopKFinalize(Vector &state_vector, AggregateFinalizeInputData &, Vect result.Verify(); } +//===--------------------------------------------------------------------===// +// State Export +//===--------------------------------------------------------------------===// +//! Exported state: STRUCT(k, values LIST(STRUCT(value, count)), filter) - the monitored values (descending count) +//! and the Filtered Space-Saving counters. Values are decoded to the input type on export, re-encoded on import. +AggregateStateLayout ApproxTopKGetStateType(AggregateLayoutInput &input) { + child_list_t value_children; + value_children.emplace_back("value", input.function.GetArguments()[0]); + value_children.emplace_back("count", LogicalType::UBIGINT); + + child_list_t children; + children.emplace_back("k", LogicalType::UBIGINT); + children.emplace_back("values", LogicalType::LIST(LogicalType::STRUCT(std::move(value_children)))); + children.emplace_back("filter", LogicalType::LIST(LogicalType::UBIGINT)); + + AggregateStateLayout layout; + layout.type = LogicalType::STRUCT(std::move(children)); + layout.total_state_size = AlignValue(sizeof(ApproxTopKState)); + return layout; +} + +template +void ApproxTopKExportState(Vector &state_vector, AggregateFinalizeInputData &aggr_input_data, Vector &result, + idx_t count, idx_t offset) { + D_ASSERT(offset == 0); + auto states = state_vector.Values(); + + auto &mask = FlatVector::ValidityMutable(result); + auto &fields = StructVector::GetEntries(result); + auto k_data = FlatVector::GetDataMutable(fields[0]); + auto &value_lists = fields[1]; + auto &filter_lists = fields[2]; + auto &k_validity = FlatVector::ValidityMutable(fields[0]); + auto &value_validity = FlatVector::ValidityMutable(value_lists); + auto &filter_validity = FlatVector::ValidityMutable(filter_lists); + auto value_entries = FlatVector::ScatterWriter(value_lists); + auto filter_entries = FlatVector::ScatterWriter(filter_lists); + idx_t total_values = ListVector::GetListSize(value_lists); + idx_t total_filters = ListVector::GetListSize(filter_lists); + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i].GetValue()->state; + value_entries[i].offset = total_values; + filter_entries[i].offset = total_filters; + if (!state_ptr || state_ptr->values.empty()) { + // no values have been added to this state - export NULL (children of a NULL struct must also be NULL) + mask.SetInvalid(i); + k_validity.SetInvalid(i); + value_validity.SetInvalid(i); + filter_validity.SetInvalid(i); + value_entries[i].length = 0; + filter_entries[i].length = 0; + k_data[i] = 0; + continue; + } + k_data[i] = state_ptr->k; + value_entries[i].length = state_ptr->values.size(); + filter_entries[i].length = state_ptr->filter.size(); + total_values += state_ptr->values.size(); + total_filters += state_ptr->filter.size(); + } + + ListVector::Reserve(value_lists, total_values); + ListVector::Reserve(filter_lists, total_filters); + auto &value_structs = ListVector::GetChildMutable(value_lists); + auto &value_fields = StructVector::GetEntries(value_structs); + auto &value_child = value_fields[0]; + auto count_data = FlatVector::GetDataMutable(value_fields[1]); + auto filter_data = FlatVector::GetDataMutable(ListVector::GetChildMutable(filter_lists)); + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i].GetValue()->state; + if (!state_ptr || state_ptr->values.empty()) { + continue; + } + auto &state = *state_ptr; + // write the values (in descending count order) - decoding them back to the input type + idx_t value_offset = value_entries[i].offset; + for (auto &val_ref : state.values) { + auto &val = val_ref.get(); + OP::template HistogramFinalize(val.str_val.str, value_child, value_offset); + count_data[value_offset] = val.count; + value_offset++; + } + for (idx_t filter_idx = 0; filter_idx < state.filter.size(); filter_idx++) { + filter_data[filter_entries[i].offset + filter_idx] = state.filter[filter_idx]; + } + } + ListVector::SetListSize(value_lists, total_values); + ListVector::SetListSize(filter_lists, total_filters); + FlatVector::SetSize(fields[0], count); + FlatVector::SetSize(value_lists, count); + FlatVector::SetSize(filter_lists, count); + FlatVector::SetSize(result, count); +} + +template +void ApproxTopKImportState(AggregateImportInputData &input) { + const auto &layout = input.layout; + const auto count = input.input_vec.size(); + // the input can be any vector type (e.g. dictionary-encoded from a compressed scan) - flatten it so the + // struct/list children can be read by position + Vector input_vec(input.input_vec, 0, count); + input_vec.Flatten(); + const auto dest_buffer = input.dest_buffer; + auto &allocator = input.allocator; + const auto validity = input_vec.Validity(); + const auto &fields = StructVector::GetEntries(input_vec); + auto k_data = FlatVector::GetData(fields[0]); + auto &value_lists = fields[1]; + auto &filter_lists = fields[2]; + auto value_entries = FlatVector::GetData(value_lists); + auto filter_entries = FlatVector::GetData(filter_lists); + auto &value_structs = ListVector::GetChild(value_lists); + const auto &value_fields = StructVector::GetEntries(value_structs); + + // encode the values - this maps them to the same representation the state stores (e.g. as sort keys) + auto extra_state = OP::CreateExtraState(); + UnifiedVectorFormat value_data; + OP::PrepareData(value_fields[0], extra_state, value_data); + auto value_strings = UnifiedVectorFormat::GetData(value_data); + auto count_data = FlatVector::GetData(value_fields[1]); + auto filter_data = FlatVector::GetData(ListVector::GetChild(filter_lists)); + + for (idx_t i = 0; i < count; i++) { + auto &state = *reinterpret_cast(dest_buffer + i * layout.total_state_size); + state.state = nullptr; + if (!validity.IsValid(i)) { + // NULL input - leave the state empty + continue; + } + auto internal_state = make_uniq(); + auto &target = *internal_state; + target.Initialize(k_data[i]); + if (value_entries[i].length > target.capacity || filter_entries[i].length != target.filter.size()) { + throw InvalidInputException("Invalid approx_top_k state - " + "the values/filter sizes do not match the k value"); + } + // insert the values - these are ordered by descending count, keeping "values" sorted + for (idx_t value_idx = 0; value_idx < value_entries[i].length; value_idx++) { + const auto idx = value_entries[i].offset + value_idx; + const auto sel_idx = value_data.sel->get_index(idx); + if (!value_data.validity.RowIsValid(sel_idx)) { + throw InvalidInputException("Invalid approx_top_k state - the state values cannot be NULL"); + } + + auto &val = target.stored_values[target.values.size()]; + val.index = target.values.size(); + target.values.push_back(val); + + const auto &str_val = value_strings[sel_idx]; + ApproxTopKString topk_string(str_val, Hash(str_val)); + InternalApproxTopKState::CopyValue(val, topk_string, allocator); + target.lookup_map.insert(make_pair(val.str_val, reference(val))); + val.count = count_data[idx]; + } + for (idx_t filter_idx = 0; filter_idx < filter_entries[i].length; filter_idx++) { + target.filter[filter_idx] = filter_data[filter_entries[i].offset + filter_idx]; + } + target.Verify(); + state.state = internal_state.release(); + } +} + unique_ptr ApproxTopKBind(BindAggregateFunctionInput &input) { auto &function = input.GetBoundFunction(); auto &arguments = input.GetArguments(); @@ -389,7 +553,11 @@ unique_ptr ApproxTopKBind(BindAggregateFunctionInput &input) { if (arguments[0]->GetReturnType().id() == LogicalTypeId::VARCHAR) { function.SetStateUpdateCallback(ApproxTopKUpdate); function.SetStateFinalizeCallback(ApproxTopKFinalize); + function.SetExportAggregateStateCallback(ApproxTopKExportState); + function.SetImportAggregateStateCallback(ApproxTopKImportState); } + // resolve the (originally ANY) value argument type so the exported state layout uses the actual input type + function.GetArguments()[0] = arguments[0]->GetReturnType(); function.SetReturnType(LogicalType::LIST(arguments[0]->GetReturnType())); return nullptr; } @@ -399,11 +567,14 @@ unique_ptr ApproxTopKBind(BindAggregateFunctionInput &input) { AggregateFunction ApproxTopKFun::GetFunction() { using STATE = ApproxTopKState; using OP = ApproxTopKOperation; - return AggregateFunction("approx_top_k", {LogicalTypeId::ANY, LogicalType::BIGINT}, - LogicalType::LIST(LogicalType::ANY), AggregateFunction::StateSize, - AggregateFunction::StateInitialize, ApproxTopKUpdate, - AggregateFunction::StateCombine, ApproxTopKFinalize, nullptr, ApproxTopKBind, - AggregateFunction::StateDestroy); + auto fun = AggregateFunction("approx_top_k", {LogicalTypeId::ANY, LogicalType::BIGINT}, + LogicalType::LIST(LogicalType::ANY), AggregateFunction::StateSize, + AggregateFunction::StateInitialize, ApproxTopKUpdate, + AggregateFunction::StateCombine, ApproxTopKFinalize, nullptr, + ApproxTopKBind, AggregateFunction::StateDestroy); + fun.SetStateExportCallbacks(ApproxTopKGetStateType, ApproxTopKExportState, + ApproxTopKImportState); + return fun; } } // namespace duckdb diff --git a/src/duckdb/extension/core_functions/aggregate/holistic/approximate_quantile.cpp b/src/duckdb/extension/core_functions/aggregate/holistic/approximate_quantile.cpp index 50aac35aa..e5fa53052 100644 --- a/src/duckdb/extension/core_functions/aggregate/holistic/approximate_quantile.cpp +++ b/src/duckdb/extension/core_functions/aggregate/holistic/approximate_quantile.cpp @@ -1,4 +1,5 @@ #include "duckdb/common/vector/flat_vector.hpp" +#include "duckdb/common/vector/vector_iterator.hpp" #include "duckdb/common/vector/list_vector.hpp" #include "duckdb/execution/expression_executor.hpp" #include "core_functions/aggregate/holistic_functions.hpp" @@ -166,6 +167,127 @@ struct ApproxQuantileScalarOperation : public ApproxQuantileOperation { } }; +//===--------------------------------------------------------------------===// +// State Export +//===--------------------------------------------------------------------===// +//! Exported state: STRUCT(count, min, max, centroids) - the value count, exact min/max and the t-digest centroids. +LogicalType ApproxQuantileExportType() { + child_list_t centroid_children; + centroid_children.emplace_back("mean", LogicalType::DOUBLE); + centroid_children.emplace_back("weight", LogicalType::DOUBLE); + + child_list_t children; + children.emplace_back("count", LogicalType::UBIGINT); + children.emplace_back("min", LogicalType::DOUBLE); + children.emplace_back("max", LogicalType::DOUBLE); + children.emplace_back("centroids", LogicalType::LIST(LogicalType::STRUCT(std::move(centroid_children)))); + return LogicalType::STRUCT(std::move(children)); +} + +//! Rebuilds the quantile parameter (e.g. 0.5 or [0.25, 0.75]) from the bind data so re-binding can supply it. +//! param_type is the declared type of the (erased) quantile argument. +Value ApproxQuantileParameterValue(const ApproximateQuantileBindData &bind_data, const LogicalType ¶m_type) { + vector quantiles; + for (auto &q : bind_data.quantiles) { + quantiles.push_back(Value::FLOAT(q)); + } + if (param_type.id() != LogicalTypeId::LIST && param_type.id() != LogicalTypeId::ARRAY) { + D_ASSERT(quantiles.size() == 1); + return quantiles[0]; + } + return Value::LIST(LogicalType::FLOAT, std::move(quantiles)); +} + +AggregateStateLayout ApproxQuantileGetStateType(AggregateLayoutInput &input) { + auto &function = input.function; + AggregateStateLayout layout; + layout.type = ApproxQuantileExportType(); + layout.total_state_size = AlignValue(sizeof(ApproxQuantileState)); + if (input.bind_data && function.GetOriginalArguments().size() == 2) { + // the quantile parameter must be a constant at bind time (its argument is erased by BindApproxQuantile) - + // record its value so that re-binding the exported state can supply it and reconstruct the bind data + auto &bind_data = input.bind_data->Cast(); + layout.constant_parameters.emplace(1, + ApproxQuantileParameterValue(bind_data, function.GetOriginalArguments()[1])); + } + return layout; +} + +//! The shape of the exported state: STRUCT(count, min, max, centroids LIST(STRUCT(mean, weight))) +using APPROX_QUANTILE_EXPORT_TYPE = + VectorStructType>>; + +void ApproxQuantileExportState(Vector &state_vector, AggregateFinalizeInputData &aggr_input_data, Vector &result, + idx_t count, idx_t offset) { + D_ASSERT(offset == 0); + auto states = state_vector.Values(); + auto writer = FlatVector::Writer(result, count); + for (idx_t i = 0; i < count; i++) { + auto &state = *states[i].GetValue(); + if (!state.h || state.pos == 0) { + // no values have been added to this state - export NULL + writer.WriteNull(); + continue; + } + // fold any unprocessed values into the centroids + state.h->compress(); + writer.WriteValue([&](auto &count_writer, auto &min_writer, auto &max_writer, auto ¢roids_writer) { + count_writer.WriteValue(state.pos); + min_writer.WriteValue(state.h->min()); + max_writer.WriteValue(state.h->max()); + auto ¢roids = state.h->processed(); + idx_t centroid_idx = 0; + for (auto ¢roid_writer : centroids_writer.WriteList(centroids.size())) { + auto ¢roid = centroids[centroid_idx++]; + centroid_writer.WriteValue([&](auto &mean_writer, auto &weight_writer) { + mean_writer.WriteValue(centroid.mean()); + weight_writer.WriteValue(centroid.weight()); + }); + } + }); + } +} + +void ApproxQuantileImportState(AggregateImportInputData &input) { + const auto &layout = input.layout; + const auto &input_vec = input.input_vec; + const auto count = input_vec.size(); + const auto dest_buffer = input.dest_buffer; + auto entries = input_vec.Values(); + for (idx_t i = 0; i < count; i++) { + auto &state = *reinterpret_cast(dest_buffer + i * layout.total_state_size); + state.h = nullptr; + state.pos = 0; + const auto entry = entries[i]; + if (!entry.IsValid()) { + // NULL input - leave the state empty + continue; + } + const auto count_entry = entry.template GetChildValue<0>(); + const auto min_entry = entry.template GetChildValue<1>(); + const auto max_entry = entry.template GetChildValue<2>(); + const auto centroid_list = entry.template GetChildValue<3>(); + if (!count_entry.IsValid() || !min_entry.IsValid() || !max_entry.IsValid() || !centroid_list.IsValid()) { + throw InvalidInputException("Invalid approx_quantile state - the state fields cannot be NULL"); + } + std::vector centroids; + centroids.reserve(centroid_list.GetListLength()); + for (const auto centroid_entry : centroid_list.GetChildValues()) { + const auto mean_entry = centroid_entry.template GetChildValue<0>(); + const auto weight_entry = centroid_entry.template GetChildValue<1>(); + if (!centroid_entry.IsValid() || !mean_entry.IsValid() || !weight_entry.IsValid()) { + throw InvalidInputException("Invalid approx_quantile state - the centroids cannot be NULL"); + } + centroids.emplace_back(mean_entry.GetValue(), weight_entry.GetValue()); + } + auto digest = make_uniq(std::move(centroids), std::vector(), + 100, 0, 0); + digest->setMinMax(min_entry.GetValue(), max_entry.GetValue()); + state.pos = count_entry.GetValue(); + state.h = digest.release(); + } +} + AggregateFunction GetApproximateQuantileAggregateFunction(const LogicalType &type) { // Not binary comparable if (type == LogicalType::TIME_TZ) { @@ -270,12 +392,16 @@ AggregateFunction ApproxQuantileDecimalFunction(const LogicalType &type) { function.SetName("approx_quantile"); function.SetSerializeCallback(ApproximateQuantileBindData::Serialize); function.SetDeserializeCallback(ApproximateQuantileBindData::Deserialize); + function.SetStateExportCallbacks(ApproxQuantileGetStateType, ApproxQuantileExportState, ApproxQuantileImportState); return function; } unique_ptr BindApproxQuantileDecimal(BindAggregateFunctionInput &input) { auto &function = input.GetBoundFunction(); auto &arguments = input.GetArguments(); + // resolve the bare DECIMAL to its actual width/scale before BindApproxQuantile records the original arguments, + // so re-binding an exported state sees a usable type (it re-specializes the impl from the recorded argument type) + function.GetArguments()[0] = arguments[0]->GetReturnType(); auto bind_data = BindApproxQuantile(input); function.ReplaceImplementation(ApproxQuantileDecimalFunction(arguments[0]->GetReturnType())); return bind_data; @@ -286,6 +412,7 @@ AggregateFunction GetApproximateQuantileAggregate(const LogicalType &type) { fun.SetBindCallback(BindApproxQuantile); fun.SetSerializeCallback(ApproximateQuantileBindData::Serialize); fun.SetDeserializeCallback(ApproximateQuantileBindData::Deserialize); + fun.SetStateExportCallbacks(ApproxQuantileGetStateType, ApproxQuantileExportState, ApproxQuantileImportState); // temporarily push an argument so we can bind the actual quantile fun.GetSignature().AddParameter(LogicalType::FLOAT); return fun; @@ -391,12 +518,15 @@ AggregateFunction ApproxQuantileDecimalListFunction(const LogicalType &type) { function.SetName("approx_quantile"); function.SetSerializeCallback(ApproximateQuantileBindData::Serialize); function.SetDeserializeCallback(ApproximateQuantileBindData::Deserialize); + function.SetStateExportCallbacks(ApproxQuantileGetStateType, ApproxQuantileExportState, ApproxQuantileImportState); return function; } unique_ptr BindApproxQuantileDecimalList(BindAggregateFunctionInput &input) { auto &function = input.GetBoundFunction(); auto &arguments = input.GetArguments(); + // resolve the bare DECIMAL before BindApproxQuantile records the original arguments (see BindApproxQuantileDecimal) + function.GetArguments()[0] = arguments[0]->GetReturnType(); auto bind_data = BindApproxQuantile(input); function.ReplaceImplementation(ApproxQuantileDecimalListFunction(arguments[0]->GetReturnType())); return bind_data; @@ -407,6 +537,7 @@ AggregateFunction GetApproxQuantileListAggregate(const LogicalType &type) { fun.SetBindCallback(BindApproxQuantile); fun.SetSerializeCallback(ApproximateQuantileBindData::Serialize); fun.SetDeserializeCallback(ApproximateQuantileBindData::Deserialize); + fun.SetStateExportCallbacks(ApproxQuantileGetStateType, ApproxQuantileExportState, ApproxQuantileImportState); // temporarily push an argument so we can bind the actual quantile auto list_of_float = LogicalType::LIST(LogicalType::FLOAT); fun.GetSignature().AddParameter(list_of_float); diff --git a/src/duckdb/extension/json/json_extension.cpp b/src/duckdb/extension/json/json_extension.cpp index a871bbba8..ca9e922ef 100644 --- a/src/duckdb/extension/json/json_extension.cpp +++ b/src/duckdb/extension/json/json_extension.cpp @@ -6,6 +6,7 @@ #include "duckdb/catalog/catalog_entry/macro_catalog_entry.hpp" #include "duckdb/catalog/default/default_functions.hpp" #include "duckdb/function/copy_function.hpp" +#include "duckdb/main/database.hpp" #include "duckdb/main/extension/extension_loader.hpp" #include "duckdb/parser/expression/function_expression.hpp" @@ -63,9 +64,11 @@ static void LoadInternal(ExtensionLoader &loader) { copy_fun.SetName("jsonl"); loader.RegisterFunction(copy_fun); - // JSON macro's + // Pass the database's ParserCache so the parser matcher is reused, not rebuilt per macro. + ParserOptions parser_options; + parser_options.parser_cache = &loader.GetDatabaseInstance().GetParserCache(); for (idx_t index = 0; JSON_MACROS[index].name != nullptr; index++) { - auto info = DefaultFunctionGenerator::CreateInternalMacroInfo(JSON_MACROS[index]); + auto info = DefaultFunctionGenerator::CreateInternalMacroInfo(JSON_MACROS[index], parser_options); loader.RegisterFunction(*info); } } diff --git a/src/duckdb/extension/parquet/column_reader.cpp b/src/duckdb/extension/parquet/column_reader.cpp index 87abad05d..e9386b59b 100644 --- a/src/duckdb/extension/parquet/column_reader.cpp +++ b/src/duckdb/extension/parquet/column_reader.cpp @@ -849,9 +849,7 @@ void ColumnReader::DirectFilter(ColumnReaderInput &input, Vector &result, const void ColumnReader::ApplyFilter(Vector &v, const TableFilter &filter, TableFilterState &filter_state, idx_t scan_count, SelectionVector &sel, idx_t &approved_tuple_count) { FlatVector::SetSize(v, count_t(scan_count)); - UnifiedVectorFormat vdata; - v.ToUnifiedFormat(vdata); - ColumnSegment::FilterSelection(sel, v, vdata, filter, filter_state, scan_count, approved_tuple_count); + ColumnSegment::FilterSelection(sel, v, filter_state, scan_count, approved_tuple_count); } void ColumnReader::Skip(idx_t num_values) { diff --git a/src/duckdb/extension/parquet/decoder/dictionary_decoder.cpp b/src/duckdb/extension/parquet/decoder/dictionary_decoder.cpp index fc3bb7cf9..a71bf4ac0 100644 --- a/src/duckdb/extension/parquet/decoder/dictionary_decoder.cpp +++ b/src/duckdb/extension/parquet/decoder/dictionary_decoder.cpp @@ -58,12 +58,9 @@ void DictionaryDecoder::InitializeDictionary(idx_t new_dictionary_size, optional filter_result = make_unsafe_uniq_array(duckdb_dictionary_size); // apply the filter - UnifiedVectorFormat vdata; - dictionary_data.ToUnifiedFormat(vdata); SelectionVector dict_sel; filter_count = duckdb_dictionary_size; - ColumnSegment::FilterSelection(dict_sel, dictionary_data, vdata, *filter, *filter_state, duckdb_dictionary_size, - filter_count); + ColumnSegment::FilterSelection(dict_sel, dictionary_data, *filter_state, duckdb_dictionary_size, filter_count); // now set all matching tuples to true for (idx_t i = 0; i < filter_count; i++) { diff --git a/src/duckdb/extension/parquet/include/parquet_prefetch_cost_model.hpp b/src/duckdb/extension/parquet/include/parquet_prefetch_cost_model.hpp index f00346028..52af9ce3f 100644 --- a/src/duckdb/extension/parquet/include/parquet_prefetch_cost_model.hpp +++ b/src/duckdb/extension/parquet/include/parquet_prefetch_cost_model.hpp @@ -34,7 +34,7 @@ class PrefetchCostModelState { public: PrefetchCostModelState() = default; - //! Refine the model from a measured network throughput estimate. + //! Refine the model from a measured throughput estimate void RefineFromEstimate(const NetworkThroughputEstimate &estimate); bool TryGetColumnGapSize(uint64_t &result) const; @@ -43,7 +43,7 @@ class PrefetchCostModelState { //! Weight applied to an estimate static constexpr double ALPHA = 0.5; - //! Measured cost model + //! Measured cost model, zero until estimates arrive so coalescing starts at the gap floor PrefetchCostModel model = {0, 0}; //! Whether we have switched from the seed to measured values bool measured_adopted = false; diff --git a/src/duckdb/extension/parquet/include/parquet_reader.hpp b/src/duckdb/extension/parquet/include/parquet_reader.hpp index 14e75d629..35c254970 100644 --- a/src/duckdb/extension/parquet/include/parquet_reader.hpp +++ b/src/duckdb/extension/parquet/include/parquet_reader.hpp @@ -188,6 +188,14 @@ struct ParquetPrefetchMetrics { } }; +//! Where the scan is in its async execution. +enum class ParquetScanState : uint8_t { + SCHEDULE, //! schedule the next row group's I/O + PROCESS, //! process the current row group into a output chunk + RESUME_PAYLOAD, //! resume decoding the payload columns after the filter-column I/O blocked + FINISHED //! the scan is done +}; + struct ParquetReaderScanState { public: ColumnReader &GetColumnReader(idx_t i); @@ -201,22 +209,17 @@ struct ParquetReaderScanState { vector> column_readers; duckdb_base_std::unique_ptr thrift_file_proto; - bool finished; + ParquetScanState scan_state; SelectionVector sel; ResizeableBuffer define_buf; ResizeableBuffer repeat_buf; bool prefetch_mode = false; - bool current_group_prefetched = false; //! Number of filter head counts, used for prefetching idx_t filter_head_count = 0; - //! true once the filters ran - bool filter_done = false; //! Surviving row count idx_t filter_count = 0; - //! Filter columns kept across the payload BLOCKED - DataChunk filter_stash; ParquetPrefetchMetrics prefetch_metrics; diff --git a/src/duckdb/extension/parquet/include/reader/expression_column_reader.hpp b/src/duckdb/extension/parquet/include/reader/expression_column_reader.hpp index 2adbb517e..1574d739d 100644 --- a/src/duckdb/extension/parquet/include/reader/expression_column_reader.hpp +++ b/src/duckdb/extension/parquet/include/reader/expression_column_reader.hpp @@ -42,12 +42,13 @@ class ExpressionColumnReader : public ColumnReader { static constexpr const PhysicalType TYPE = PhysicalType::INVALID; public: - ExpressionColumnReader(ClientContext &context, unique_ptr child_reader, unique_ptr expr, - const ParquetColumnSchema &schema); - ExpressionColumnReader(ClientContext &context, unique_ptr child_reader, unique_ptr expr, - unique_ptr owned_schema); + ExpressionColumnReader(ClientContext &context, vector> child_readers, + unique_ptr expr, const ParquetColumnSchema &schema); + ExpressionColumnReader(ClientContext &context, vector> child_readers, + unique_ptr expr, unique_ptr owned_schema); - unique_ptr child_reader; + //! Reader(s) to produce the input(s) for the expression + vector> child_readers; DataChunk intermediate_chunk; unique_ptr expr; ExpressionExecutor executor; @@ -64,16 +65,25 @@ class ExpressionColumnReader : public ColumnReader { idx_t GroupRowsAvailable() override; uint64_t TotalCompressedSize() override { - return child_reader->TotalCompressedSize(); + idx_t total_compressed_size = 0; + for (auto &child_reader : child_readers) { + total_compressed_size += child_reader->TotalCompressedSize(); + } + return total_compressed_size; } idx_t FileOffset() const override { - return child_reader->FileOffset(); + return child_readers[0]->FileOffset(); } void RegisterPrefetch(ThriftFileTransport &transport, bool allow_merge) override { - child_reader->RegisterPrefetch(transport, allow_merge); + for (auto &child_reader : child_readers) { + child_reader->RegisterPrefetch(transport, allow_merge); + } } + +private: + void InitializeChunk(); }; } // namespace duckdb diff --git a/src/duckdb/extension/parquet/parquet_extension.cpp b/src/duckdb/extension/parquet/parquet_extension.cpp index f0821f14d..961390d04 100644 --- a/src/duckdb/extension/parquet/parquet_extension.cpp +++ b/src/duckdb/extension/parquet/parquet_extension.cpp @@ -1001,8 +1001,12 @@ static void LoadInternal(ExtensionLoader &loader) { config.AddExtensionOption("disable_parquet_prefetching", "Disable the prefetching mechanism in Parquet", LogicalType::BOOLEAN, Value(false)); config.AddExtensionOption("prefetch_all_parquet_files", - "Use the prefetching mechanism for all types of parquet files", LogicalType::BOOLEAN, - Value(false)); + "(deprecated) Parquet files are now always prefetched, this setting has no effect", + LogicalType::BOOLEAN, Value(false)); + config.AddExtensionOption( + "parquet_prefetch_column_gap", + "Byte gap under which Parquet prefetch I/O ranges are coalesced (NULL lets the cost model adapt it)", + LogicalType::UBIGINT, Value(LogicalType::UBIGINT)); config.AddExtensionOption("parquet_metadata_cache", "Cache Parquet metadata - useful when reading the same files multiple times", LogicalType::BOOLEAN, Value(false)); diff --git a/src/duckdb/extension/parquet/parquet_geometry.cpp b/src/duckdb/extension/parquet/parquet_geometry.cpp index a5c7adc29..63ae814b3 100644 --- a/src/duckdb/extension/parquet/parquet_geometry.cpp +++ b/src/duckdb/extension/parquet/parquet_geometry.cpp @@ -396,7 +396,9 @@ unique_ptr GeometryColumnReader::Create(const ParquetReader &reade func.SetName("ST_GeomFromWKB"); auto read_expr = func.Bind(context, std::move(args)); auto type_expr = BoundCastExpression::AddDefaultCastToType(std::move(read_expr), schema.type); - return make_uniq(context, std::move(string_reader), std::move(type_expr), schema); + vector> expression_children; + expression_children.push_back(std::move(string_reader)); + return make_uniq(context, std::move(expression_children), std::move(type_expr), schema); } } // namespace duckdb diff --git a/src/duckdb/extension/parquet/parquet_reader.cpp b/src/duckdb/extension/parquet/parquet_reader.cpp index 71631df59..04357972c 100644 --- a/src/duckdb/extension/parquet/parquet_reader.cpp +++ b/src/duckdb/extension/parquet/parquet_reader.cpp @@ -23,7 +23,6 @@ #include "duckdb/common/encryption_state.hpp" #include "duckdb/common/helper.hpp" #include "duckdb/common/string_util.hpp" -#include "duckdb/common/vector_operations/vector_operations.hpp" #include "duckdb/planner/table_filter.hpp" #include "duckdb/storage/object_cache.hpp" #include "duckdb/planner/table_filter_state.hpp" @@ -208,12 +207,29 @@ CreateThriftFileProtocol(QueryContext context, CachingFileHandle &file_handle, b static bool ShouldAndCanPrefetch(ClientContext &context, CachingFileHandle &file_handle) { Value disable_prefetch = false; - Value prefetch_all_files = false; context.TryGetCurrentSetting("disable_parquet_prefetching", disable_prefetch); - context.TryGetCurrentSetting("prefetch_all_parquet_files", prefetch_all_files); - bool should_prefetch = !file_handle.OnDiskFile() || prefetch_all_files.GetValue(); - bool can_prefetch = file_handle.CanSeek() && !disable_prefetch.GetValue(); - return should_prefetch && can_prefetch; + // local files also prefetch by default, the async I/O overlaps with decoding + return file_handle.CanSeek() && !disable_prefetch.GetValue(); +} + +//! Coalescing gap for the scan's prefetch I/O, either pinned through a setting or chosen by the cost model +static uint64_t DetermineAcceptedColumnGap(ClientContext &context, ParquetReaderScanState &state) { + uint64_t gap = ReadHeadComparator::DEFAULT_ACCEPTED_COLUMN_GAP; + if (!state.prefetch_mode) { + return gap; + } + Value pinned_gap; + context.TryGetCurrentSetting("parquet_prefetch_column_gap", pinned_gap); + if (!pinned_gap.IsNull()) { + return MinValue(pinned_gap.GetValue(), PrefetchCostModel::GAP_MAX); + } + // remote files measure their requests in the file system, local files in the caching file handle + NetworkThroughputEstimate estimate; + if (state.file_handle->TryGetNetworkThroughput(estimate)) { + state.cost_model_state.RefineFromEstimate(estimate); + } + state.cost_model_state.TryGetColumnGapSize(gap); + return gap; } static void ParseParquetFooter(const_data_ptr_t buffer, const string &file_path, idx_t file_size, @@ -624,8 +640,11 @@ unique_ptr ParquetReader::CreateReaderRecursive(ClientContext &con auto expr_schema = make_uniq( ParquetColumnSchema::FromParentSchema(column_reader->Schema(), cast_expression->GetReturnType(), ParquetColumnSchemaType::EXPRESSION)); + + vector> expression_children; + expression_children.push_back(std::move(column_reader)); auto expr_reader = make_uniq( - context, std::move(column_reader), std::move(cast_expression), std::move(expr_schema)); + context, std::move(expression_children), std::move(cast_expression), std::move(expr_schema)); return std::move(expr_reader); } return column_reader; @@ -1429,8 +1448,9 @@ ParquetScanFilter::~ParquetScanFilter() { void ParquetReader::InitializeScan(ClientContext &context, ParquetReaderScanState &state, vector groups_to_read) const { state.current_group = -1; - state.finished = false; + state.scan_state = ParquetScanState::SCHEDULE; state.offset_in_group = 0; + state.filter_count = 0; state.group_idx_list = std::move(groups_to_read); state.sel.Initialize(STANDARD_VECTOR_SIZE); if (!state.file_handle || state.file_handle->GetPath() != file_handle->GetPath()) { @@ -1459,32 +1479,33 @@ void ParquetReader::InitializeScan(ClientContext &context, ParquetReaderScanStat state.filter_eliminated_all_rows.assign(state.scan_filters.size(), false); } - uint64_t accepted_column_gap = ReadHeadComparator::DEFAULT_ACCEPTED_COLUMN_GAP; - if (state.prefetch_mode && !state.file_handle->OnDiskFile()) { - NetworkThroughputEstimate estimate; - if (state.file_handle->TryGetNetworkThroughput(estimate)) { - state.cost_model_state.RefineFromEstimate(estimate); - } - state.cost_model_state.TryGetColumnGapSize(accepted_column_gap); - } - state.thrift_file_proto = - CreateThriftFileProtocol(context, *state.file_handle, state.prefetch_mode, accepted_column_gap); + state.thrift_file_proto = CreateThriftFileProtocol(context, *state.file_handle, state.prefetch_mode, + DetermineAcceptedColumnGap(context, state)); state.column_readers.resize(column_indexes.size()); for (idx_t i = 0; i < column_indexes.size(); i++) { auto &index = column_indexes[i]; auto column_id = index.GetPrimaryIndex(); - auto &schema = root_schema->children[column_id]; - auto column_reader = CreateReaderRecursive(context, index, schema); auto it = expression_map.find(column_id); if (it != expression_map.end()) { - auto &expression = it->second; + auto &expression_data = it->second; + auto &expression = expression_data.expression; + auto &expression_column_indexes = expression_data.column_indexes; + D_ASSERT(!expression_column_indexes.empty()); + + vector> child_readers; + for (auto &id : expression_column_indexes) { + auto &schema = root_schema->children[id.GetPrimaryIndex()]; + child_readers.push_back(CreateReaderRecursive(context, id, schema)); + } auto expr_schema = make_uniq(ParquetColumnSchema::FromParentSchema( - column_reader->Schema(), expression->GetReturnType(), ParquetColumnSchemaType::EXPRESSION)); - auto expr_reader = make_uniq(context, std::move(column_reader), expression->Copy(), + child_readers[0]->Schema(), expression->GetReturnType(), ParquetColumnSchemaType::EXPRESSION)); + auto expr_reader = make_uniq(context, std::move(child_readers), expression->Copy(), std::move(expr_schema)); state.column_readers[i] = std::move(expr_reader); } else { + auto &schema = root_schema->children[column_id]; + auto column_reader = CreateReaderRecursive(context, index, schema); state.column_readers[i] = std::move(column_reader); } } @@ -1600,13 +1621,10 @@ CollectIOTasks(ThriftFileTransport &trans, const shared_ptr & ParquetPrefetchStrategy ParquetReader::WholeGroupPrefetch(ParquetReaderScanState &state, ThriftFileTransport &trans, const duckdb_parquet::RowGroup &group, uint64_t total_row_group_span, bool log_prefetch) { - if (!state.current_group_prefetched) { - auto total_compressed_size = GetGroupCompressedSize(state); - if (total_compressed_size > 0) { - trans.RegisterPrefetch(GetGroupOffset(state), total_row_group_span, false); - trans.FinalizeRegistration(); - } - state.current_group_prefetched = true; + auto total_compressed_size = GetGroupCompressedSize(state); + if (total_compressed_size > 0) { + trans.RegisterPrefetch(GetGroupOffset(state), total_row_group_span, false); + trans.FinalizeRegistration(); } if (log_prefetch) { state.prefetch_metrics.logger.strategy = ParquetPrefetchStrategy::WHOLE_GROUP; @@ -1693,20 +1711,24 @@ ParquetPrefetchStrategy ParquetReader::ColumnWisePrefetch(ParquetReaderScanState } AsyncResult ParquetReader::Scan(ClientContext &context, ParquetReaderScanState &state, DataChunk &result) { - result.Reset(); - if (state.finished) { - return SourceResultType::FINISHED; - } - const bool log_prefetch = Logger::Get(context).ShouldLog(ParquetPrefetchLogType::NAME, ParquetPrefetchLogType::LEVEL); - // see if we have to switch to the next row group in the parquet file - if (state.current_group < 0 || (int64_t)state.offset_in_group >= GetGroup(state).num_rows) { + switch (state.scan_state) { + case ParquetScanState::FINISHED: + result.Reset(); + return SourceResultType::FINISHED; + case ParquetScanState::SCHEDULE: + result.Reset(); return Schedule(context, state, result, log_prefetch); + case ParquetScanState::PROCESS: + result.Reset(); + return Process(state, result, log_prefetch); + case ParquetScanState::RESUME_PAYLOAD: + return Process(state, result, log_prefetch); + default: + throw InternalException("Unexpected ParquetScanState"); } - - return Process(state, result, log_prefetch); } AsyncResult ParquetReader::Schedule(ClientContext &context, ParquetReaderScanState &state, DataChunk &result, @@ -1716,17 +1738,10 @@ AsyncResult ParquetReader::Schedule(ClientContext &context, ParquetReaderScanSta auto &trans = reinterpret_cast(*state.thrift_file_proto->getTransport()); trans.ClearPrefetch(); - state.current_group_prefetched = false; state.filter_head_count = 0; - if (state.prefetch_mode && !state.file_handle->OnDiskFile()) { - NetworkThroughputEstimate estimate; - if (state.file_handle->TryGetNetworkThroughput(estimate)) { - state.cost_model_state.RefineFromEstimate(estimate); - } - uint64_t accepted_column_gap = ReadHeadComparator::DEFAULT_ACCEPTED_COLUMN_GAP; - state.cost_model_state.TryGetColumnGapSize(accepted_column_gap); - trans.SetAcceptedColumnGap(accepted_column_gap); + if (state.prefetch_mode) { + trans.SetAcceptedColumnGap(DetermineAcceptedColumnGap(context, state)); } if (log_prefetch && state.prefetch_metrics.filter_ran && state.current_group > 0) { @@ -1735,7 +1750,7 @@ AsyncResult ParquetReader::Schedule(ClientContext &context, ParquetReaderScanSta state.prefetch_metrics.FinalizeRowGroupSelectivity(); if ((idx_t)state.current_group == state.group_idx_list.size()) { - state.finished = true; + state.scan_state = ParquetScanState::FINISHED; return SourceResultType::FINISHED; } @@ -1769,55 +1784,63 @@ AsyncResult ParquetReader::Schedule(ClientContext &context, ParquetReaderScanSta double scan_percentage = (double)(to_scan_compressed_bytes) / static_cast(total_row_group_span); if (to_scan_compressed_bytes > total_row_group_span) { - throw IOException( - "The parquet file '%s' seems to have incorrectly set page offsets. This interferes with DuckDB's " - "prefetching optimization. DuckDB may still be able to scan this file by manually disabling the " - "prefetching mechanism using: 'SET disable_parquet_prefetching=true'.", - GetFileName()); - } - // We basically do eager fetch if we do a whole-group or column-wise prefetch, if we do filters we do it lazily - ParquetPrefetchStrategy strategy = ParquetPrefetchStrategy::NONE; - if (parquet_options.prefetch_strategy == ParquetPrefetchStrategyOption::WHOLE_GROUP) { - strategy = WholeGroupPrefetch(state, trans, group, total_row_group_span, log_prefetch); - } else { - bool filters_look_unselective = false; - if (filters) { - if (state.prefetch_metrics.row_groups_executed == 0) { - filters_look_unselective = true; - } else if (static_cast(state.prefetch_metrics.row_groups_with_matches) / - static_cast(state.prefetch_metrics.row_groups_executed) > - ParquetReaderPrefetchConfig::PREFETCH_FILTER_MINIMUM_MATCH_RATIO) { - filters_look_unselective = true; - } + if (!state.file_handle->OnDiskFile()) { + throw IOException( + "The parquet file '%s' seems to have incorrectly set page offsets. This interferes with DuckDB's " + "prefetching optimization. DuckDB may still be able to scan this file by manually disabling the " + "prefetching mechanism using: 'SET disable_parquet_prefetching=true'.", + GetFileName()); } - - if ((!filters || filters_look_unselective) && - scan_percentage > ParquetReaderPrefetchConfig::WHOLE_GROUP_PREFETCH_MINIMUM_SCAN) { + // broken page offsets cannot be prefetched, local files fall back to synchronous reads + state.prefetch_mode = false; + } + if (state.prefetch_mode) { + // whole group and column wise prefetch fetch eagerly, filter prefetch fetches lazily + ParquetPrefetchStrategy strategy = ParquetPrefetchStrategy::NONE; + if (parquet_options.prefetch_strategy == ParquetPrefetchStrategyOption::WHOLE_GROUP) { strategy = WholeGroupPrefetch(state, trans, group, total_row_group_span, log_prefetch); } else { - strategy = ColumnWisePrefetch(state, trans, group, filters_look_unselective, log_prefetch); + bool filters_look_unselective = false; + if (filters) { + if (state.prefetch_metrics.row_groups_executed == 0) { + filters_look_unselective = true; + } else if (static_cast(state.prefetch_metrics.row_groups_with_matches) / + static_cast(state.prefetch_metrics.row_groups_executed) > + ParquetReaderPrefetchConfig::PREFETCH_FILTER_MINIMUM_MATCH_RATIO) { + filters_look_unselective = true; + } + } + + if ((!filters || filters_look_unselective) && + scan_percentage > ParquetReaderPrefetchConfig::WHOLE_GROUP_PREFETCH_MINIMUM_SCAN) { + strategy = WholeGroupPrefetch(state, trans, group, total_row_group_span, log_prefetch); + } else { + strategy = ColumnWisePrefetch(state, trans, group, filters_look_unselective, log_prefetch); + } + } + auto read_head_count = trans.GetReadHeads().size(); + switch (strategy) { + case ParquetPrefetchStrategy::PREFETCH_FILTERS: + // schedule only the filter columns' I/O, they are last in our list + io_tasks = CollectIOTasks(trans, state.file_handle, read_head_count - state.filter_head_count, + read_head_count); + break; + case ParquetPrefetchStrategy::WHOLE_GROUP: + case ParquetPrefetchStrategy::COLUMN_WISE_EAGER: + // schedule the I/O for all columns up front + io_tasks = CollectIOTasks(trans, state.file_handle, 0, read_head_count); + break; + default: + throw InternalException("Unexpected parquet prefetch strategy when scheduling I/O"); + } + if (log_prefetch) { + state.prefetch_metrics.logger.accepted_column_gap = trans.GetAcceptedColumnGap(); } - } - auto read_head_count = trans.GetReadHeads().size(); - switch (strategy) { - case ParquetPrefetchStrategy::PREFETCH_FILTERS: - // schedule only the filter columns' I/O, they are last in our list - io_tasks = - CollectIOTasks(trans, state.file_handle, read_head_count - state.filter_head_count, read_head_count); - break; - case ParquetPrefetchStrategy::WHOLE_GROUP: - case ParquetPrefetchStrategy::COLUMN_WISE_EAGER: - // schedule the I/O for all columns up front - io_tasks = CollectIOTasks(trans, state.file_handle, 0, read_head_count); - break; - default: - throw InternalException("Unexpected parquet prefetch strategy when scheduling I/O"); - } - if (log_prefetch) { - state.prefetch_metrics.logger.accepted_column_gap = trans.GetAcceptedColumnGap(); } } result.Reset(); + // a skipped row group has no rows to decode, so we loop back to schedule the next one + state.scan_state = row_group_skipped ? ParquetScanState::SCHEDULE : ParquetScanState::PROCESS; if (!io_tasks.empty()) { return AsyncResult(std::move(io_tasks), TaskSchedulerType::ASYNC); } @@ -1914,18 +1937,6 @@ void ParquetReader::DecodeRemainingColumns(ParquetReaderScanState &state, DataCh } } -// When doing ParquetPrefetchStrategy::PREFETCH_FILTERS, we gotta block mid processing to get the other columns. -// on the re-entry the chunk is reset in the multifilescan which borkes our datachunk, hence we need to copy for -// ownership. -// FIXME: maybe we can change this to be able to move? or not have the multifile reset? bigger refactor though. -static void CopyFilterColumns(const vector &scan_filters, DataChunk &src, DataChunk &dst, - idx_t count) { - for (auto &scan_filter : scan_filters) { - auto idx = MultiFileLocalIndex(scan_filter.filter_idx).GetIndex(); - VectorOperations::Copy(src.data[idx], dst.data[idx], count, 0, 0); - } -} - vector> ParquetReader::ScheduleRemainingColumns(ParquetReaderScanState &state, DataChunk &result, idx_t scan_count) { if (state.filter_count == 0 || state.filter_head_count == 0) { @@ -1939,25 +1950,13 @@ vector> ParquetReader::ScheduleRemainingColumns(ParquetRea // no payload to do return {}; } - // stash the decoded filter columns and empty the chunk for the BLOCKED return - if (state.filter_stash.data.empty()) { - state.filter_stash.Initialize(allocator, result.GetTypes()); - } - state.filter_stash.Reset(); - CopyFilterColumns(state.scan_filters, result, state.filter_stash, scan_count); - result.Reset(); - state.filter_done = true; + state.scan_state = ParquetScanState::RESUME_PAYLOAD; return io_tasks; } AsyncResult ParquetReader::ProcessFilters(ParquetReaderScanState &state, DataChunk &result, idx_t scan_count, uint8_t *define_ptr, uint8_t *repeat_ptr, bool log_prefetch) { - if (state.filter_done) { - // the filters already ran, we restore the stashed columns - state.filter_done = false; - CopyFilterColumns(state.scan_filters, state.filter_stash, result, scan_count); - } else { - // we need to evaluate the filters, then async-fetch the payload and resume into the decode + if (state.scan_state == ParquetScanState::PROCESS) { state.filter_count = EvaluateFilters(state, result, scan_count, define_ptr, repeat_ptr, log_prefetch); auto io_tasks = ScheduleRemainingColumns(state, result, scan_count); if (!io_tasks.empty()) { @@ -1972,11 +1971,14 @@ AsyncResult ParquetReader::ProcessFilters(ParquetReaderScanState &state, DataChu } AsyncResult ParquetReader::Process(ParquetReaderScanState &state, DataChunk &result, bool log_prefetch) { - auto scan_count = MinValue(STANDARD_VECTOR_SIZE, GetGroup(state).num_rows - state.offset_in_group); - result.SetChildCardinality(scan_count); + const idx_t group_num_rows = GetGroup(state).num_rows; + auto scan_count = MinValue(STANDARD_VECTOR_SIZE, group_num_rows - state.offset_in_group); + if (state.scan_state == ParquetScanState::PROCESS) { + result.SetChildCardinality(scan_count); + } if (scan_count == 0) { - state.finished = true; + state.scan_state = ParquetScanState::FINISHED; // end of last group, we are done return SourceResultType::FINISHED; } @@ -2017,6 +2019,8 @@ AsyncResult ParquetReader::Process(ParquetReaderScanState &state, DataChunk &res result.SetChildCardinality(result.size()); rows_read += scan_count; state.offset_in_group += scan_count; + // once the group is fully consumed we schedule the next one, otherwise we keep processing this group + state.scan_state = state.offset_in_group >= group_num_rows ? ParquetScanState::SCHEDULE : ParquetScanState::PROCESS; return SourceResultType::HAVE_MORE_OUTPUT; } diff --git a/src/duckdb/extension/parquet/reader/expression_column_reader.cpp b/src/duckdb/extension/parquet/reader/expression_column_reader.cpp index e599b5610..5e8cdf1f0 100644 --- a/src/duckdb/extension/parquet/reader/expression_column_reader.cpp +++ b/src/duckdb/extension/parquet/reader/expression_column_reader.cpp @@ -23,45 +23,72 @@ class ClientContext; //===--------------------------------------------------------------------===// // Expression Column Reader //===--------------------------------------------------------------------===// -ExpressionColumnReader::ExpressionColumnReader(ClientContext &context, unique_ptr child_reader_p, +ExpressionColumnReader::ExpressionColumnReader(ClientContext &context, vector> child_readers_p, unique_ptr expr_p, const ParquetColumnSchema &schema_p) - : ColumnReader(child_reader_p->Reader(), schema_p), child_reader(std::move(child_reader_p)), + : ColumnReader(child_readers_p[0]->Reader(), schema_p), child_readers(std::move(child_readers_p)), expr(std::move(expr_p)), executor(context, expr.get()) { - vector intermediate_types {child_reader->Type()}; - intermediate_chunk.Initialize(reader.allocator, intermediate_types); + if (child_readers.empty()) { + throw InternalException("Can't instantiate an ExpressionColumnReader with 0 children"); + } + InitializeChunk(); } -ExpressionColumnReader::ExpressionColumnReader(ClientContext &context, unique_ptr child_reader_p, +ExpressionColumnReader::ExpressionColumnReader(ClientContext &context, vector> child_readers_p, unique_ptr expr_p, unique_ptr owned_schema_p) - : ColumnReader(child_reader_p->Reader(), *owned_schema_p), child_reader(std::move(child_reader_p)), + : ColumnReader(child_readers_p[0]->Reader(), *owned_schema_p), child_readers(std::move(child_readers_p)), expr(std::move(expr_p)), executor(context, expr.get()), owned_schema(std::move(owned_schema_p)) { - vector intermediate_types {child_reader->Type()}; + if (child_readers.empty()) { + throw InternalException("Can't instantiate an ExpressionColumnReader with 0 children"); + } + InitializeChunk(); +} + +void ExpressionColumnReader::InitializeChunk() { + vector intermediate_types; + for (auto &child_reader : child_readers) { + intermediate_types.push_back(child_reader->Type()); + }; intermediate_chunk.Initialize(reader.allocator, intermediate_types); } void ExpressionColumnReader::InitializeRead(idx_t row_group_idx_p, const vector &columns, TProtocol &protocol_p) { - child_reader->InitializeRead(row_group_idx_p, columns, protocol_p); + for (auto &child_reader : child_readers) { + child_reader->InitializeRead(row_group_idx_p, columns, protocol_p); + } } idx_t ExpressionColumnReader::Read(ColumnReaderInput &input, Vector &result) { intermediate_chunk.Reset(); - auto &intermediate_vector = intermediate_chunk.data[0]; - auto amount = child_reader->Read(input, intermediate_vector); + optional_idx amount; + for (idx_t i = 0; i < child_readers.size(); i++) { + auto &intermediate_vector = intermediate_chunk.data[i]; + auto &child_reader = child_readers[i]; + + auto res = child_reader->Read(input, intermediate_vector); + if (amount.IsValid() && res != amount.GetIndex()) { + throw InternalException("ExpressionColumnReader children Read calls produced differing amounts (%d and %d)", + res, amount.GetIndex()); + } + amount = res; + } + // Execute the expression - intermediate_chunk.SetChildCardinality(amount); + intermediate_chunk.SetChildCardinality(amount.GetIndex()); executor.ExecuteExpression(intermediate_chunk, result); - return amount; + return amount.GetIndex(); } void ExpressionColumnReader::Skip(idx_t num_values) { - child_reader->Skip(num_values); + for (auto &child_reader : child_readers) { + child_reader->Skip(num_values); + } } idx_t ExpressionColumnReader::GroupRowsAvailable() { - return child_reader->GroupRowsAvailable(); + return child_readers[0]->GroupRowsAvailable(); } } // namespace duckdb diff --git a/src/duckdb/src/catalog/default/default_functions.cpp b/src/duckdb/src/catalog/default/default_functions.cpp index afe503e16..454802727 100644 --- a/src/duckdb/src/catalog/default/default_functions.cpp +++ b/src/duckdb/src/catalog/default/default_functions.cpp @@ -4,6 +4,8 @@ #include "duckdb/parser/statement/create_statement.hpp" #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp" #include "duckdb/common/string_util.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/main/database.hpp" namespace duckdb { @@ -222,11 +224,16 @@ static const DefaultMacro internal_macros[] = { {nullptr, nullptr, nullptr}}; unique_ptr DefaultFunctionGenerator::CreateInternalMacroInfo(const DefaultMacro &default_macro) { + return CreateInternalMacroInfo(default_macro, ParserOptions()); +} + +unique_ptr DefaultFunctionGenerator::CreateInternalMacroInfo(const DefaultMacro &default_macro, + ParserOptions options) { auto bind_info = make_uniq(CatalogType::MACRO_ENTRY); // Build a full CREATE MACRO statement and let the parser handle parameters, types, and defaults. // macro_definition may contain multiple comma-separated overloads, e.g. "(x) AS x, (x, y) AS x+y". auto sql = StringUtil::Format("CREATE MACRO __dummy__%s", default_macro.macro_definition); - Parser parser; + Parser parser(options); parser.ParseQuery(sql); D_ASSERT(parser.statements.size() == 1); D_ASSERT(parser.statements[0]->type == StatementType::CREATE_STATEMENT); @@ -254,12 +261,13 @@ static bool DefaultFunctionMatches(const DefaultMacro ¯o, const Identifier & return macro.schema == schema && macro.name == name; } -static unique_ptr GetDefaultFunction(const Identifier &input_schema, const Identifier &input_name) { +static unique_ptr GetDefaultFunction(const Identifier &input_schema, const Identifier &input_name, + ParserOptions options) { auto &schema = input_schema; auto &name = input_name; for (idx_t index = 0; internal_macros[index].name != nullptr; index++) { if (DefaultFunctionMatches(internal_macros[index], schema, name)) { - return DefaultFunctionGenerator::CreateInternalMacroInfo(internal_macros[index]); + return DefaultFunctionGenerator::CreateInternalMacroInfo(internal_macros[index], options); } } return nullptr; @@ -271,7 +279,9 @@ DefaultFunctionGenerator::DefaultFunctionGenerator(Catalog &catalog, SchemaCatal unique_ptr DefaultFunctionGenerator::CreateDefaultEntry(ClientContext &context, const Identifier &entry_name) { - auto info = GetDefaultFunction(schema.name, entry_name); + ParserOptions options; + options.parser_cache = &context.db->GetParserCache(); + auto info = GetDefaultFunction(schema.name, entry_name, options); if (info) { return make_uniq_base(catalog, schema, info->Cast()); } diff --git a/src/duckdb/src/catalog/default/default_table_functions.cpp b/src/duckdb/src/catalog/default/default_table_functions.cpp index e2bc4f584..5f5c403e5 100644 --- a/src/duckdb/src/catalog/default/default_table_functions.cpp +++ b/src/duckdb/src/catalog/default/default_table_functions.cpp @@ -4,6 +4,8 @@ #include "duckdb/parser/parsed_data/create_macro_info.hpp" #include "duckdb/parser/statement/select_statement.hpp" #include "duckdb/function/table_macro_function.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/main/database.hpp" namespace duckdb { @@ -118,7 +120,12 @@ DefaultTableFunctionGenerator::CreateInternalTableMacroInfo(const DefaultTableMa unique_ptr DefaultTableFunctionGenerator::CreateTableMacroInfo(const DefaultTableMacro &default_macro) { - Parser parser; + return CreateTableMacroInfo(default_macro, ParserOptions()); +} + +unique_ptr DefaultTableFunctionGenerator::CreateTableMacroInfo(const DefaultTableMacro &default_macro, + ParserOptions options) { + Parser parser(options); parser.ParseQuery(default_macro.macro); if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { throw InternalException("Expected a single select statement in CreateTableMacroInfo internal"); @@ -130,10 +137,10 @@ DefaultTableFunctionGenerator::CreateTableMacroInfo(const DefaultTableMacro &def } static unique_ptr GetDefaultTableFunction(const Identifier &input_schema, - const Identifier &input_name) { + const Identifier &input_name, ParserOptions options) { for (idx_t index = 0; internal_table_macros[index].name != nullptr; index++) { if (internal_table_macros[index].schema == input_schema && internal_table_macros[index].name == input_name) { - return DefaultTableFunctionGenerator::CreateTableMacroInfo(internal_table_macros[index]); + return DefaultTableFunctionGenerator::CreateTableMacroInfo(internal_table_macros[index], options); } } return nullptr; @@ -141,7 +148,9 @@ static unique_ptr GetDefaultTableFunction(const Identifier & unique_ptr DefaultTableFunctionGenerator::CreateDefaultEntry(ClientContext &context, const Identifier &entry_name) { - auto info = GetDefaultTableFunction(schema.name, entry_name); + ParserOptions options; + options.parser_cache = &context.db->GetParserCache(); + auto info = GetDefaultTableFunction(schema.name, entry_name, options); if (info) { return make_uniq_base(catalog, schema, info->Cast()); } diff --git a/src/duckdb/src/common/enum_util.cpp b/src/duckdb/src/common/enum_util.cpp index 5547f7de3..ff0a4fd37 100644 --- a/src/duckdb/src/common/enum_util.cpp +++ b/src/duckdb/src/common/enum_util.cpp @@ -29,6 +29,7 @@ #include "duckdb/common/enums/cte_materialize.hpp" #include "duckdb/common/enums/date_part_specifier.hpp" #include "duckdb/common/enums/debug_initialize.hpp" +#include "duckdb/common/enums/debug_order_verification.hpp" #include "duckdb/common/enums/debug_statement_verification.hpp" #include "duckdb/common/enums/debug_vector_verification.hpp" #include "duckdb/common/enums/debug_verification_mode.hpp" @@ -1593,6 +1594,25 @@ DebugInitialize EnumUtil::FromString(const char *value) { return static_cast(StringUtil::StringToEnum(GetDebugInitializeValues(), 3, "DebugInitialize", value)); } +const StringUtil::EnumStringLiteral *GetDebugOrderVerificationValues() { + static constexpr StringUtil::EnumStringLiteral values[] { + { static_cast(DebugOrderVerification::NONE), "NONE" }, + { static_cast(DebugOrderVerification::CREATE_SORT_KEY), "CREATE_SORT_KEY" }, + { static_cast(DebugOrderVerification::VARIANT), "VARIANT" } + }; + return values; +} + +template<> +const char* EnumUtil::ToChars(DebugOrderVerification value) { + return StringUtil::EnumToString(GetDebugOrderVerificationValues(), 3, "DebugOrderVerification", static_cast(value)); +} + +template<> +DebugOrderVerification EnumUtil::FromString(const char *value) { + return static_cast(StringUtil::StringToEnum(GetDebugOrderVerificationValues(), 3, "DebugOrderVerification", value)); +} + const StringUtil::EnumStringLiteral *GetDebugStatementVerificationValues() { static constexpr StringUtil::EnumStringLiteral values[] { { static_cast(DebugStatementVerification::NONE), "NONE" }, diff --git a/src/duckdb/src/common/multi_file/multi_file_column_mapper.cpp b/src/duckdb/src/common/multi_file/multi_file_column_mapper.cpp index 269af0035..67378a196 100644 --- a/src/duckdb/src/common/multi_file/multi_file_column_mapper.cpp +++ b/src/duckdb/src/common/multi_file/multi_file_column_mapper.cpp @@ -635,6 +635,7 @@ ResultColumnMapping MultiFileColumnMapper::CreateColumnMappingByMapper(const Col auto global_column_id = global_id.GetPrimaryIndex(); optional_ptr global_column_reference; + optional_ptr global_column_p; auto local_idx = MultiFileLocalIndex(reader.column_ids.size()); if (IsVirtualColumn(global_column_id)) { // virtual column - look it up in the virtual column entry map @@ -653,33 +654,42 @@ ResultColumnMapping MultiFileColumnMapper::CreateColumnMappingByMapper(const Col } // the column is not constant for the file // get the expression to evaluate the column OR the global column to read into - auto expr = - multi_file_reader.GetVirtualColumnExpression(context, reader_data, local_columns, global_column_id, - virtual_column_type, local_idx, global_column_reference); - if ((!expr && !global_column_reference) || (expr && global_column_reference.get())) { - throw InternalException(R"( - The GetVirtualColumnExpression is expected to either:" - - return an expression applied in FinalizeChunk to create the value for this global column, - forwarding the (potentially changed) 'global_column_id' to the reader to create the needed data for the expression. - - set the 'global_column_reference' to replace this virtual column with a MultiFileColumnDefinition, as if it was defined in the schema. - Doing neither or both is not a valid option. - )"); - } - if (expr && expr->GetExpressionType() == ExpressionType::VALUE_CONSTANT) { - // the column is constant after all - handle it - expressions.push_back(std::move(expr)); + auto virtual_column_bind_result = multi_file_reader.GetVirtualColumnExpression( + context, reader_data, local_columns, global_column_id, virtual_column_type, local_idx); + auto bind_result_type = virtual_column_bind_result.GetType(); + switch (bind_result_type) { + case MultiFileReaderVirtualColumnBinding::VirtualColumnBindingType::CONSTANT: { + expressions.push_back(make_uniq(virtual_column_bind_result.constant)); continue; } - if (!global_column_reference) { + case MultiFileReaderVirtualColumnBinding::VirtualColumnBindingType::EXPRESSION: { + auto &expr = virtual_column_bind_result.expression; + auto &column_ids = virtual_column_bind_result.local_virtual_column_ids; + auto is_reference = expr->GetExpressionType() == ExpressionType::BOUND_REF; expressions.push_back(std::move(expr)); - MultiFileLocalColumnId local_id(reader.columns.size()); - auto local_index = global_id.RemapRootIndex(local_id.GetId()); - // add the virtual column to the reader - reader.columns.emplace_back(virtual_entry->second.name.GetIdentifierName(), virtual_column_type); - reader.AddVirtualColumn(global_column_id); + bool seen_virtual_column = false; + for (auto &id : column_ids) { + if (!IsVirtualColumn(id)) { + reader.column_ids.push_back(MultiFileLocalColumnId(id)); + reader.column_indexes.push_back(global_id.RemapRootIndex(id)); + continue; + } + if (seen_virtual_column) { + throw InternalException("GetVirtualColumnExpression is only allowed to return one virtual " + "column id for the EXPRESSION result"); + } + seen_virtual_column = true; + MultiFileLocalColumnId local_id(reader.columns.size()); + auto local_index = global_id.RemapRootIndex(local_id.GetId()); + + reader.columns.emplace_back(virtual_entry->second.name.GetIdentifierName(), virtual_column_type); + reader.column_ids.push_back(local_id); + reader.column_indexes.push_back(std::move(local_index)); + reader.AddVirtualColumn(id); + } // set it as being projected in this spot MultiFileColumnMap index_mapping(local_idx, virtual_column_type, virtual_column_type); @@ -687,14 +697,18 @@ ResultColumnMapping MultiFileColumnMapper::CreateColumnMappingByMapper(const Col index_mapping.filter_conversion = FilterConversionType::CANNOT_CONVERT; } result.global_to_local.insert(make_pair(global_idx, std::move(index_mapping))); - reader.column_ids.push_back(local_id); - reader.column_indexes.push_back(std::move(local_index)); continue; } + case MultiFileReaderVirtualColumnBinding::VirtualColumnBindingType::COLUMN_REFERENCE: { + global_column_p = virtual_column_bind_result.global_column_reference; + break; + } + } + } else { + global_column_p = global_columns[global_column_id]; } - const auto &global_column = - global_column_reference ? *global_column_reference : global_columns[global_column_id]; + const auto &global_column = *global_column_p; if (reader.UseCastMap()) { // reader is responsible for converting types - perform a top-level match only auto entry = mapper.Find(global_column); @@ -1046,21 +1060,23 @@ static unique_ptr TryCastTableFilter(const TableFilter &global_filt return make_uniq(std::move(rewritten_expr)); } -static void SetIndexToZero(unique_ptr &root_expr) { -#ifdef DEBUG - optional_idx index; - ExpressionIterator::VisitExpressionMutable(root_expr, [&](BoundReferenceExpression &ref, - unique_ptr &expr) { - if (index.IsValid() && index.GetIndex() != ref.Index()) { - throw InternalException("Expected an expression that only references a single column, but found multiple!"); - } - index = ref.Index(); - ref.IndexMutable() = 0; - }); -#else +static idx_t StartIndexAtZero(unique_ptr &root_expr) { + map>> reference_map; + + //! First gather all references, sorted by their index ExpressionIterator::VisitExpressionMutable( - root_expr, [&](BoundReferenceExpression &ref, unique_ptr &expr) { ref.IndexMutable() = 0; }); -#endif + root_expr, [&reference_map](BoundReferenceExpression &ref, unique_ptr &expr) { + reference_map[ref.Index()].push_back(ref); + }); + idx_t new_index = 0; + //! Then assign all the references a new index, starting at 0 + for (auto &[_, references] : reference_map) { + auto index = new_index++; + for (auto &ref : references) { + ref.get().IndexMutable() = index; + } + } + return new_index; } unique_ptr @@ -1073,6 +1089,8 @@ MultiFileColumnMapper::CreateFilters(map(); map local_to_global; + + idx_t local_offset = 0; for (auto &it : filters) { auto &global_index = it.first; auto &global_filter = it.second.get().Cast(); @@ -1083,7 +1101,7 @@ MultiFileColumnMapper::CreateFilters(mapsecond; - auto local_id = map_entry.mapping.index; + auto local_id = MultiFileLocalIndex(map_entry.mapping.index.GetIndex() - local_offset); auto filter_idx = reader.column_indexes[local_id].GetPrimaryIndex(); auto &local_type = map_entry.local_type; auto &global_type = map_entry.global_type; @@ -1112,8 +1130,15 @@ MultiFileColumnMapper::CreateFilters(map expression_column_indexes; + for (idx_t i = 0; i < unique_ref_count; i++) { + auto index = local_id + i; + expression_column_indexes.push_back(reader.column_indexes[index]); + } + reader.expression_map.emplace(filter_idx, + BaseFileReaderExpression(std::move(expr), expression_column_indexes)); // reset the expression - since we are evaluating it in the reader we can just reference it expr = make_uniq(global_type, local_id); diff --git a/src/duckdb/src/common/multi_file/multi_file_reader.cpp b/src/duckdb/src/common/multi_file/multi_file_reader.cpp index b2175e373..8a21fcdb6 100644 --- a/src/duckdb/src/common/multi_file/multi_file_reader.cpp +++ b/src/duckdb/src/common/multi_file/multi_file_reader.cpp @@ -541,12 +541,12 @@ unique_ptr MultiFileReader::GetConstantVirtualColumn(MultiFileReader return nullptr; } -unique_ptr MultiFileReader::GetVirtualColumnExpression(ClientContext &, MultiFileReaderData &, - const vector &, - idx_t &column_id, const LogicalType &type, - MultiFileLocalIndex local_idx, - optional_ptr &) { - return make_uniq(type, local_idx.GetIndex()); +MultiFileReaderVirtualColumnBinding +MultiFileReader::GetVirtualColumnExpression(ClientContext &, MultiFileReaderData &, + const vector &, idx_t column_id, + const LogicalType &type, MultiFileLocalIndex local_idx) { + return MultiFileReaderVirtualColumnBinding(make_uniq(type, local_idx.GetIndex()), + {column_id}); } MultiFileReaderBindData MultiFileReader::BindUnionReader(ClientContext &context, vector &return_types, diff --git a/src/duckdb/src/common/types/variant/variant_iterator.cpp b/src/duckdb/src/common/types/variant/variant_iterator.cpp new file mode 100644 index 000000000..bd52b81f8 --- /dev/null +++ b/src/duckdb/src/common/types/variant/variant_iterator.cpp @@ -0,0 +1,529 @@ +#include "duckdb/common/types/variant_iterator.hpp" +#include "duckdb/common/vector/shredded_vector.hpp" +#include "duckdb/common/vector/list_vector.hpp" +#include "duckdb/common/vector/array_vector.hpp" +#include "duckdb/common/vector/struct_vector.hpp" +#include "duckdb/common/serializer/varint.hpp" + +#include + +namespace duckdb { + +//! Indices into the shredded "STRUCT(typed_value, untyped_value_index)" wrapper +static constexpr idx_t TYPED_VALUE_INDEX = 0; +static constexpr idx_t UNTYPED_VALUE_INDEX = 1; + +//! Child indices into the unshredded layout (see UnshreddedVariantLayout) +static constexpr idx_t KEYS_INDEX = 0; +static constexpr idx_t CHILDREN_INDEX = 1; +static constexpr idx_t VALUES_INDEX = 2; +static constexpr idx_t DATA_INDEX = 3; +//! Child indices within the children / values structs +static constexpr idx_t KEY_ID_INDEX = 0; +static constexpr idx_t VALUE_ID_INDEX = 1; +static constexpr idx_t TYPE_ID_INDEX = 0; +static constexpr idx_t BYTE_OFFSET_INDEX = 1; + +namespace { + +//! Decode the (length-prefixed) string payload of a value at the given byte offset in the blob +string_t DecodeStringData(const string_t &blob, uint32_t byte_offset) { + auto ptr = const_data_ptr_cast(blob.GetData()) + byte_offset; + auto length = VarintDecode(ptr); + return string_t(reinterpret_cast(ptr), length); +} + +//! Decode the (width, scale, value pointer) of a DECIMAL value at the given byte offset +VariantDecimalData DecodeDecimalData(const string_t &blob, uint32_t byte_offset) { + auto ptr = const_data_ptr_cast(blob.GetData()) + byte_offset; + auto width = VarintDecode(ptr); + auto scale = VarintDecode(ptr); + return VariantDecimalData(width, scale, ptr); +} + +//! Decode the (child_count, children_idx) of an OBJECT/ARRAY value at the given byte offset +VariantNestedData DecodeNestedData(const string_t &blob, uint32_t byte_offset) { + auto ptr = const_data_ptr_cast(blob.GetData()) + byte_offset; + VariantNestedData result; + result.child_count = VarintDecode(ptr); + result.children_idx = result.child_count ? VarintDecode(ptr) : 0; + return result; +} + +//! Whether the value at logical position 'index' of a shredded layer is valid (non-NULL) +bool ShreddedIsValid(const ShreddedVariantIterator &node, idx_t index) { + return node.unified.validity.RowIsValid(node.unified.sel->get_index(index)); +} + +} // namespace + +//===--------------------------------------------------------------------===// +// VariantIteratorState +//===--------------------------------------------------------------------===// +VariantIteratorState::VariantIteratorState(const Vector &variant) + //! The unshredded ("core") source is the variant itself, or the unshredded component of a shredded vector + : unshredded(variant.GetVectorType() == VectorType::SHREDDED_VECTOR ? ShreddedVector::GetUnshreddedVector(variant) + : variant) { + if (variant.GetVectorType() != VectorType::SHREDDED_VECTOR) { + return; + } + is_shredded = true; + + //! Build the (recursive) view of the shredded component. This does not reconstruct the unshredded + //! representation (unlike flattening the SHREDDED_VECTOR itself) - it only normalizes the regular, + //! typed vectors of the shredded tree so each layer can be navigated by index. + auto &shredded_vec = ShreddedVector::GetShreddedVector(variant); + shredded_format.Build(shredded_vec); +} + +//===--------------------------------------------------------------------===// +// ShreddedVariantIterator +//===--------------------------------------------------------------------===// +void ShreddedVariantIterator::Build(const Vector &vec) { + vec.ToUnifiedFormat(unified); + logical_type = vec.GetType(); + + switch (vec.GetType().InternalType()) { + case PhysicalType::LIST: + children.resize(1); + children[0].Build(ListVector::GetChild(vec)); + break; + case PhysicalType::ARRAY: + children.resize(1); + children[0].Build(ArrayVector::GetChild(vec)); + break; + case PhysicalType::STRUCT: { + auto &entries = StructVector::GetEntries(vec); + children.resize(entries.size()); + for (idx_t i = 0; i < entries.size(); i++) { + children[i].Build(entries[i]); + } + break; + } + default: + break; + } +} + +//===--------------------------------------------------------------------===// +// UnshreddedVariantIterator +//===--------------------------------------------------------------------===// +UnshreddedVariantIterator::UnshreddedVariantIterator(const Vector &unshredded) : data(unshredded) { +} + +bool UnshreddedVariantIterator::RowIsValid(idx_t row) const { + return data[row].IsValid(); +} + +VariantLogicalType UnshreddedVariantIterator::GetTypeId(idx_t row, idx_t value_index) const { + auto raw = data[row].GetChildValue().GetChildValue(value_index).GetChildValue(); + return static_cast(raw.GetValueUnsafe()); +} + +uint32_t UnshreddedVariantIterator::GetByteOffset(idx_t row, idx_t value_index) const { + return data[row] + .GetChildValue() + .GetChildValue(value_index) + .GetChildValue() + .GetValueUnsafe(); +} + +const string_t &UnshreddedVariantIterator::GetBlob(idx_t row) const { + return data[row].GetChildValue().GetValueUnsafe(); +} + +string_t UnshreddedVariantIterator::GetKey(idx_t row, idx_t key_index) const { + return data[row].GetChildValue().GetChildValue(key_index).GetValueUnsafe(); +} + +uint32_t UnshreddedVariantIterator::GetKeysIndex(idx_t row, idx_t child_index) const { + return data[row] + .GetChildValue() + .GetChildValue(child_index) + .GetChildValue() + .GetValueUnsafe(); +} + +uint32_t UnshreddedVariantIterator::GetValuesIndex(idx_t row, idx_t child_index) const { + return data[row] + .GetChildValue() + .GetChildValue(child_index) + .GetChildValue() + .GetValueUnsafe(); +} + +//===--------------------------------------------------------------------===// +// Root / row validity +//===--------------------------------------------------------------------===// +VariantIterator VariantIteratorState::Root(idx_t row) const { + if (is_shredded) { + //! The shredded component's top-level validity is the authoritative row validity (a SQL-NULL row + //! has the whole shredded struct set to NULL). This must be checked separately because + //! ResolveShredded only inspects the typed_value / untyped_value_index of a wrapper, never the + //! wrapper's own struct validity. + if (!ShreddedIsValid(shredded_format, row)) { + return VariantIterator::MakeNull(*this); + } + auto root = VariantIterator::ResolveShredded(*this, shredded_format, row, row); + //! a root value is never "missing" - treat any such case as a SQL NULL + return root.IsMissing() ? VariantIterator::MakeNull(*this) : root; + } + if (!unshredded.RowIsValid(row)) { + return VariantIterator::MakeNull(*this); + } + //! The unshredded root value lives at values[0] + return VariantIterator::MakeUnshredded(*this, row, 0); +} + +bool VariantIteratorState::RowIsValid(idx_t row) const { + //! A VARIANT is never NULL at the root via a VARIANT_NULL value (that is reserved for nested values) - + //! a root that resolves to NULL is a genuine SQL NULL. This matches the semantics of unshredding, + //! where a shredded value whose typed leaf is NULL (with no unshredded leftover) becomes a SQL NULL. + return !Root(row).IsNull(); +} + +//===--------------------------------------------------------------------===// +// VariantIterator - factory helpers +//===--------------------------------------------------------------------===// +VariantIterator VariantIterator::MakeNull(const VariantIteratorState &state) { + VariantIterator result; + result.state = &state; + result.kind = Kind::NULL_VALUE; + return result; +} + +VariantIterator VariantIterator::MakeMissing(const VariantIteratorState &state) { + VariantIterator result; + result.state = &state; + result.kind = Kind::MISSING; + return result; +} + +VariantIterator VariantIterator::MakeUnshredded(const VariantIteratorState &state, idx_t row, uint32_t value_index) { + VariantIterator result; + result.state = &state; + result.kind = Kind::UNSHREDDED; + result.row = row; + result.value_index = value_index; + return result; +} + +VariantIterator VariantIterator::MakeShredded(const VariantIteratorState &state, const ShreddedVariantIterator &content, + idx_t index, idx_t row, uint32_t overlay_value_index) { + VariantIterator result; + result.state = &state; + result.kind = Kind::SHREDDED; + result.row = row; + result.shredded_format = content; + result.shredded_index = index; + result.overlay_value_index = overlay_value_index; + return result; +} + +//===--------------------------------------------------------------------===// +// Shredded resolution +//===--------------------------------------------------------------------===// +VariantIterator VariantIterator::ResolveShredded(const VariantIteratorState &state, const ShreddedVariantIterator &node, + idx_t index, idx_t row) { + if (node.logical_type.id() != LogicalTypeId::STRUCT) { + //! A flattened (fully-consistent) primitive - a NULL here represents a VARIANT_NULL value + if (!ShreddedIsValid(node, index)) { + return MakeNull(state); + } + return MakeShredded(state, node, index, row, 0); + } + + //! A "STRUCT(typed_value, [untyped_value_index])" wrapper + auto &typed_value = node.children[TYPED_VALUE_INDEX]; + const bool has_overlay = node.children.size() > 1; + + if (ShreddedIsValid(typed_value, index)) { + //! The value is (at least partially) shredded. The untyped_value_index (overlay) is only relevant + //! for OBJECTs (to merge leftover fields) - for primitives and ARRAYs we avoid reading it entirely, + //! which is the hot path for fully-shredded values. + if (!has_overlay || typed_value.logical_type.id() != LogicalTypeId::STRUCT) { + return MakeShredded(state, typed_value, index, row, 0); + } + //! OBJECT: read the overlay - either NULL (fully shredded) or a 1-based index to the leftover object + auto &untyped = node.children[UNTYPED_VALUE_INDEX]; + auto untyped_sel = untyped.unified.sel->get_index(index); + uint32_t overlay = 0; + if (untyped.unified.validity.RowIsValid(untyped_sel)) { + overlay = untyped.unified.GetData()[untyped_sel]; + } + return MakeShredded(state, typed_value, index, row, overlay); + } + + //! The typed value is absent - the value (if any) lives entirely in the unshredded component + if (!has_overlay) { + //! No unshredded value -> VARIANT_NULL + return MakeNull(state); + } + auto &untyped = node.children[UNTYPED_VALUE_INDEX]; + auto untyped_sel = untyped.unified.sel->get_index(index); + if (!untyped.unified.validity.RowIsValid(untyped_sel)) { + //! NULL untyped value -> VARIANT_NULL + return MakeNull(state); + } + auto overlay_value_index = untyped.unified.GetData()[untyped_sel]; + if (overlay_value_index == 0) { + //! 0 is reserved to indicate a missing value + return MakeMissing(state); + } + //! The unshredded component stores 1-based value indices (0 == missing) + return MakeUnshredded(state, row, overlay_value_index - 1); +} + +//===--------------------------------------------------------------------===// +// Type resolution +//===--------------------------------------------------------------------===// +static VariantLogicalType ShreddedTypeId(const ShreddedVariantIterator &content, idx_t index) { + auto &type = content.logical_type; + switch (type.id()) { + case LogicalTypeId::STRUCT: + return VariantLogicalType::OBJECT; + case LogicalTypeId::LIST: + return VariantLogicalType::ARRAY; + case LogicalTypeId::BOOLEAN: + return content.unified.GetData()[content.unified.sel->get_index(index)] ? VariantLogicalType::BOOL_TRUE + : VariantLogicalType::BOOL_FALSE; + case LogicalTypeId::TINYINT: + return VariantLogicalType::INT8; + case LogicalTypeId::SMALLINT: + return VariantLogicalType::INT16; + case LogicalTypeId::INTEGER: + return VariantLogicalType::INT32; + case LogicalTypeId::BIGINT: + return VariantLogicalType::INT64; + case LogicalTypeId::HUGEINT: + return VariantLogicalType::INT128; + case LogicalTypeId::UTINYINT: + return VariantLogicalType::UINT8; + case LogicalTypeId::USMALLINT: + return VariantLogicalType::UINT16; + case LogicalTypeId::UINTEGER: + return VariantLogicalType::UINT32; + case LogicalTypeId::UBIGINT: + return VariantLogicalType::UINT64; + case LogicalTypeId::UHUGEINT: + return VariantLogicalType::UINT128; + case LogicalTypeId::FLOAT: + return VariantLogicalType::FLOAT; + case LogicalTypeId::DOUBLE: + return VariantLogicalType::DOUBLE; + case LogicalTypeId::DECIMAL: + return VariantLogicalType::DECIMAL; + case LogicalTypeId::VARCHAR: + return VariantLogicalType::VARCHAR; + case LogicalTypeId::BLOB: + return VariantLogicalType::BLOB; + case LogicalTypeId::UUID: + return VariantLogicalType::UUID; + case LogicalTypeId::DATE: + return VariantLogicalType::DATE; + case LogicalTypeId::TIME: + return VariantLogicalType::TIME_MICROS; + case LogicalTypeId::TIME_NS: + return VariantLogicalType::TIME_NANOS; + case LogicalTypeId::TIME_TZ: + return VariantLogicalType::TIME_MICROS_TZ; + case LogicalTypeId::TIMESTAMP_SEC: + return VariantLogicalType::TIMESTAMP_SEC; + case LogicalTypeId::TIMESTAMP_MS: + return VariantLogicalType::TIMESTAMP_MILIS; + case LogicalTypeId::TIMESTAMP: + return VariantLogicalType::TIMESTAMP_MICROS; + case LogicalTypeId::TIMESTAMP_NS: + return VariantLogicalType::TIMESTAMP_NANOS; + case LogicalTypeId::TIMESTAMP_TZ: + return VariantLogicalType::TIMESTAMP_MICROS_TZ; + case LogicalTypeId::TIMESTAMP_TZ_NS: + return VariantLogicalType::TIMESTAMP_NANOS_TZ; + case LogicalTypeId::INTERVAL: + return VariantLogicalType::INTERVAL; + case LogicalTypeId::BIGNUM: + return VariantLogicalType::BIGNUM; + case LogicalTypeId::BIT: + return VariantLogicalType::BITSTRING; + case LogicalTypeId::GEOMETRY: + return VariantLogicalType::GEOMETRY; + default: + throw NotImplementedException("Shredded VARIANT type '%s' is not supported by VariantIterator", + type.ToString()); + } +} + +VariantLogicalType VariantIterator::GetTypeId() const { + switch (kind) { + case Kind::NULL_VALUE: + return VariantLogicalType::VARIANT_NULL; + case Kind::UNSHREDDED: + return state->unshredded.GetTypeId(row, value_index); + case Kind::SHREDDED: + return ShreddedTypeId(*shredded_format, shredded_index); + default: + throw InternalException("VariantIterator::GetTypeId called on a MISSING value"); + } +} + +//===--------------------------------------------------------------------===// +// Primitive accessors +//===--------------------------------------------------------------------===// +const_data_ptr_t VariantIterator::GetDataPointer() const { + if (kind == Kind::UNSHREDDED) { + auto &blob = state->unshredded.GetBlob(row); + return const_data_ptr_cast(blob.GetData()) + state->unshredded.GetByteOffset(row, value_index); + } + D_ASSERT(kind == Kind::SHREDDED); + auto &content = *shredded_format; + auto type_size = GetTypeIdSize(content.unified.physical_type); + return content.unified.data + content.unified.sel->get_index(shredded_index) * type_size; +} + +string_t VariantIterator::GetString() const { + if (kind == Kind::UNSHREDDED) { + return DecodeStringData(state->unshredded.GetBlob(row), state->unshredded.GetByteOffset(row, value_index)); + } + D_ASSERT(kind == Kind::SHREDDED); + auto &content = *shredded_format; + return content.unified.GetData()[content.unified.sel->get_index(shredded_index)]; +} + +VariantDecimalData VariantIterator::GetDecimal() const { + if (kind == Kind::UNSHREDDED) { + return DecodeDecimalData(state->unshredded.GetBlob(row), state->unshredded.GetByteOffset(row, value_index)); + } + D_ASSERT(kind == Kind::SHREDDED); + auto &content = *shredded_format; + auto &type = content.logical_type; + auto width = DecimalType::GetWidth(type); + auto scale = DecimalType::GetScale(type); + auto type_size = GetTypeIdSize(content.unified.physical_type); + auto value_ptr = content.unified.data + content.unified.sel->get_index(shredded_index) * type_size; + return VariantDecimalData(width, scale, value_ptr); +} + +//===--------------------------------------------------------------------===// +// Nested accessors +//===--------------------------------------------------------------------===// +VariantObjectIterator VariantIterator::GetObjectChildren(VariantIterationOrder order) const { + return VariantObjectIterator(*this, order); +} + +VariantArrayIterator VariantIterator::GetArrayChildren() const { + return VariantArrayIterator(*this); +} + +//===--------------------------------------------------------------------===// +// VariantArrayIterator +//===--------------------------------------------------------------------===// +VariantArrayIterator::VariantArrayIterator(const VariantIterator &array) + : state(array.state), row(array.row), shredded(array.kind == VariantIterator::Kind::SHREDDED) { + if (!shredded) { + auto nested = + DecodeNestedData(state->unshredded.GetBlob(row), state->unshredded.GetByteOffset(row, array.value_index)); + base = nested.children_idx; + length = nested.child_count; + return; + } + //! The array's elements live at flat positions (offset + i) in the list's child layer + auto &content = *array.shredded_format; + auto list_data = content.unified.GetData(); + auto &entry = list_data[content.unified.sel->get_index(array.shredded_index)]; + base = entry.offset; + length = entry.length; + element_node = content.children[0]; +} + +VariantIterator VariantArrayIterator::operator[](idx_t i) const { + if (shredded) { + return VariantIterator::ResolveShredded(*state, *element_node, base + i, row); + } + return VariantIterator::MakeUnshredded(*state, row, state->unshredded.GetValuesIndex(row, base + i)); +} + +//===--------------------------------------------------------------------===// +// VariantObjectIterator +//===--------------------------------------------------------------------===// +VariantObjectIterator::VariantObjectIterator(const VariantIterator &object, VariantIterationOrder order) + : state(object.state), row(object.row), order(order), shredded(object.kind == VariantIterator::Kind::SHREDDED) { + if (!shredded) { + auto nested = + DecodeNestedData(state->unshredded.GetBlob(row), state->unshredded.GetByteOffset(row, object.value_index)); + base = nested.children_idx; + raw_count = nested.child_count; + } else { + //! Shredded object: the typed struct fields, followed by the leftover (overlay) object's fields + content = object.shredded_format; + shredded_index = object.shredded_index; + typed_field_count = content->children.size(); + raw_count = typed_field_count; + if (object.overlay_value_index != 0) { + auto overlay_value_index = object.overlay_value_index - 1; + auto nested = DecodeNestedData(state->unshredded.GetBlob(row), + state->unshredded.GetByteOffset(row, overlay_value_index)); + overlay_base = nested.children_idx; + raw_count += nested.child_count; + } + } + + if (order != VariantIterationOrder::LEXICOGRAPHIC) { + return; + } + + //! Materialize all (non-missing) entries and sort them by key up-front + ordered_entries.reserve(raw_count); + for (idx_t raw_pos = 0; raw_pos < raw_count; raw_pos++) { + auto entry = RawEntry(raw_pos); + if (entry.value.IsMissing()) { + continue; + } + ordered_entries.push_back(entry); + } + std::sort(ordered_entries.begin(), ordered_entries.end(), + [](const VariantObjectEntry &a, const VariantObjectEntry &b) { return a.key < b.key; }); +} + +VariantObjectEntry VariantObjectIterator::RawEntry(idx_t raw_pos) const { + if (!shredded) { + auto child_idx = base + raw_pos; + auto key_idx = state->unshredded.GetKeysIndex(row, child_idx); + auto value_idx = state->unshredded.GetValuesIndex(row, child_idx); + return VariantObjectEntry {state->unshredded.GetKey(row, key_idx), + VariantIterator::MakeUnshredded(*state, row, value_idx)}; + } + if (raw_pos < typed_field_count) { + //! A shredded (typed) field - struct fields preserve the (logical) index of the parent + auto &name = StructType::GetChildTypes(content->logical_type)[raw_pos].first; + return VariantObjectEntry { + string_t(name.c_str(), NumericCast(name.size())), + VariantIterator::ResolveShredded(*state, content->children[raw_pos], shredded_index, row)}; + } + //! A leftover field from the overlay (unshredded) object + auto child_idx = overlay_base + (raw_pos - typed_field_count); + auto key_idx = state->unshredded.GetKeysIndex(row, child_idx); + auto value_idx = state->unshredded.GetValuesIndex(row, child_idx); + return VariantObjectEntry {state->unshredded.GetKey(row, key_idx), + VariantIterator::MakeUnshredded(*state, row, value_idx)}; +} + +void VariantObjectIterator::Iterator::Load() { + if (parent.order == VariantIterationOrder::LEXICOGRAPHIC) { + //! Iterate the pre-sorted entries (missing fields were already filtered out) + if (pos < parent.ordered_entries.size()) { + current = parent.ordered_entries[pos]; + } + return; + } + //! INTERNAL order: lazily materialize, skipping missing entries (only shredded typed fields can be + //! "missing", i.e. absent for this row) + while (pos < parent.raw_count) { + current = parent.RawEntry(pos); + if (!current.value.IsMissing()) { + return; + } + ++pos; + } +} + +} // namespace duckdb diff --git a/src/duckdb/src/common/vector/shredded_vector.cpp b/src/duckdb/src/common/vector/shredded_vector.cpp index faee370f0..89c4096f5 100644 --- a/src/duckdb/src/common/vector/shredded_vector.cpp +++ b/src/duckdb/src/common/vector/shredded_vector.cpp @@ -59,6 +59,20 @@ Value ShreddedVectorBuffer::GetValue(const LogicalType &type, idx_t index) const return result_vec.GetValue(0); } +buffer_ptr ShreddedVectorBuffer::SliceInternal(const LogicalType &type, idx_t offset, idx_t end) { + // propagate the slice into the shredded data and emit a new shredded vector + auto count = count_t(end - offset); + Vector sliced(*shredded_data, offset, end); + return make_buffer(sliced, count); +} + +buffer_ptr ShreddedVectorBuffer::SliceInternal(const LogicalType &type, const SelectionVector &sel, + idx_t count) { + // propagate the slice into the shredded data and emit a new shredded vector + Vector sliced(*shredded_data, sel, count); + return make_buffer(sliced, count_t(count)); +} + buffer_ptr ShreddedVectorBuffer::FlattenSliceInternal(const LogicalType &type, const SelectionVector &sel, idx_t count) const { Vector *source = shredded_data.get(); diff --git a/src/duckdb/src/common/vector/struct_vector.cpp b/src/duckdb/src/common/vector/struct_vector.cpp index 48df3c1c5..23be50c57 100644 --- a/src/duckdb/src/common/vector/struct_vector.cpp +++ b/src/duckdb/src/common/vector/struct_vector.cpp @@ -293,6 +293,10 @@ vector &StructVector::GetEntries(Vector &vector) { if (vector.GetVectorType() == VectorType::DICTIONARY_VECTOR) { throw InternalException("Struct vectors cannot be dictionary vectors"); } + if (vector.GetVectorType() != VectorType::FLAT_VECTOR && vector.GetVectorType() != VectorType::CONSTANT_VECTOR) { + // non-flat representation (e.g. a shredded variant) - flatten so we can access the struct entries + vector.Flatten(); + } D_ASSERT(vector.GetVectorType() == VectorType::FLAT_VECTOR || vector.GetVectorType() == VectorType::CONSTANT_VECTOR); D_ASSERT(vector.GetBufferRef()); diff --git a/src/duckdb/src/common/vector_operations/comparison_operators.cpp b/src/duckdb/src/common/vector_operations/comparison_operators.cpp index deae898ed..db33f0e73 100644 --- a/src/duckdb/src/common/vector_operations/comparison_operators.cpp +++ b/src/duckdb/src/common/vector_operations/comparison_operators.cpp @@ -536,57 +536,6 @@ static void ArrayComparator(const Vector &left, const Vector &right, int8_t *res ListOrArrayComparator(left, right, result_data, lhs_sel, rhs_sel, sel_count, accessor, result_validity); } -static void VariantComparator(const Vector &left, const Vector &right, int8_t *result_data, - const SelectionVector &lhs_sel, const SelectionVector &rhs_sel, idx_t sel_count, - optional_ptr result_validity = nullptr) { - RecursiveUnifiedVectorFormat left_recursive_data, right_recursive_data; - Vector::RecursiveToUnifiedFormat(left, left_recursive_data); - Vector::RecursiveToUnifiedFormat(right, right_recursive_data); - - UnifiedVariantVectorData left_variant(left_recursive_data); - UnifiedVariantVectorData right_variant(right_recursive_data); - - auto &left_data = left_recursive_data.unified; - auto &right_data = right_recursive_data.unified; - for (idx_t i = 0; i < sel_count; i++) { - auto left_idx = left_data.sel->get_index(lhs_sel.get_index(i)); - auto right_idx = right_data.sel->get_index(rhs_sel.get_index(i)); - - bool left_null = !left_data.validity.RowIsValid(left_idx); - bool right_null = !right_data.validity.RowIsValid(right_idx); - - if (left_null || right_null) { - if (!result_validity) { - // DISTINCT - result_data[i] = DistinctNullComparator(left_null, right_null); - } else { - // regular comparison - set NULL if any value is NULL - result_validity->SetInvalid(i); - } - continue; - } - - // both non-NULL: convert to Values and compare - auto left_val = VariantUtils::ConvertVariantToValue(left_variant, lhs_sel.get_index(i), 0); - auto right_val = VariantUtils::ConvertVariantToValue(right_variant, rhs_sel.get_index(i), 0); - - LogicalType max_logical_type; - if (!LogicalType::DefaultTryGetMaxLogicalTypeUnchecked(left_val.type(), right_val.type(), max_logical_type)) { - throw InvalidInputException( - "Can't compare values of type %s (%s) and type %s (%s) - an explicit cast is required", - left_val.type().ToString(), left_val.ToString(), right_val.type().ToString(), right_val.ToString()); - } - - if (ValueOperations::DistinctGreaterThan(left_val, right_val)) { - result_data[i] = Comparator::LEFT_IS_GREATER; - } else if (ValueOperations::DistinctGreaterThan(right_val, left_val)) { - result_data[i] = Comparator::RIGHT_IS_GREATER; - } else { - result_data[i] = Comparator::VALUES_ARE_EQUAL; - } - } -} - static void DistinctComparatorTypeSwitchInternal(const Vector &left, const Vector &right, int8_t *result_data, const SelectionVector &lhs_sel, const SelectionVector &rhs_sel, idx_t sel_count) { @@ -636,11 +585,7 @@ static void DistinctComparatorTypeSwitchInternal(const Vector &left, const Vecto DistinctComparatorExecute::Execute(left, right, result_data, lhs_sel, rhs_sel, sel_count); break; case PhysicalType::STRUCT: - if (left.GetType().id() == LogicalTypeId::VARIANT) { - VariantComparator(left, right, result_data, lhs_sel, rhs_sel, sel_count); - } else { - StructComparator(left, right, result_data, lhs_sel, rhs_sel, sel_count); - } + StructComparator(left, right, result_data, lhs_sel, rhs_sel, sel_count); break; case PhysicalType::LIST: ListComparator(left, right, result_data, lhs_sel, rhs_sel, sel_count); @@ -714,9 +659,7 @@ static void ComparatorTypeSwitch(const Vector &left, const Vector &right, Vector auto &validity = FlatVector::ValidityMutable(result); auto &sel = *FlatVector::IncrementalSelectionVector(); auto physical_type = left.GetType().InternalType(); - if (physical_type == PhysicalType::STRUCT && left.GetType().id() == LogicalTypeId::VARIANT) { - VariantComparator(left, right, result_data, sel, sel, count, validity); - } else if (physical_type == PhysicalType::STRUCT) { + if (physical_type == PhysicalType::STRUCT) { StructComparator(left, right, result_data, sel, sel, count, validity); } else if (physical_type == PhysicalType::LIST) { ListComparator(left, right, result_data, sel, sel, count, validity); diff --git a/src/duckdb/src/function/aggregate_function.cpp b/src/duckdb/src/function/aggregate_function.cpp index 89af82665..e204b78fd 100644 --- a/src/duckdb/src/function/aggregate_function.cpp +++ b/src/duckdb/src/function/aggregate_function.cpp @@ -64,7 +64,8 @@ bool AggregateFunctionCallbacks::operator==(const AggregateFunctionCallbacks &rh init_local_state_finalize == rhs.init_local_state_finalize && cluster_update == rhs.cluster_update && window == rhs.window && window_init == rhs.window_init && window_batch == rhs.window_batch && bind == rhs.bind && destructor == rhs.destructor && statistics == rhs.statistics && - serialize == rhs.serialize && deserialize == rhs.deserialize && get_state_type == rhs.get_state_type; + serialize == rhs.serialize && deserialize == rhs.deserialize && get_state_type == rhs.get_state_type && + export_aggregate_state == rhs.export_aggregate_state && import_aggregate_state == rhs.import_aggregate_state; } bool AggregateFunctionCallbacks::operator!=(const AggregateFunctionCallbacks &rhs) const { diff --git a/src/duckdb/src/function/cast/variant/to_variant.cpp b/src/duckdb/src/function/cast/variant/to_variant.cpp index f0525ecfe..4255ac751 100644 --- a/src/duckdb/src/function/cast/variant/to_variant.cpp +++ b/src/duckdb/src/function/cast/variant/to_variant.cpp @@ -111,7 +111,7 @@ struct VariantCastData : BoundCastData { LogicalType shredded_type; unique_ptr Copy() const override { - return make_uniq(shredded_type); + return make_uniq(*this); } }; diff --git a/src/duckdb/src/function/function_list.cpp b/src/duckdb/src/function/function_list.cpp index 878d2b0be..b094cf3a8 100644 --- a/src/duckdb/src/function/function_list.cpp +++ b/src/duckdb/src/function/function_list.cpp @@ -262,6 +262,7 @@ static const StaticFunctionDefinition function[] = { DUCKDB_SCALAR_FUNCTION_ALIAS(UcaseFun), DUCKDB_SCALAR_FUNCTION(UpperFun), DUCKDB_SCALAR_FUNCTION_SET(VariantArrayLengthFun), + DUCKDB_SCALAR_FUNCTION(VariantComparatorFun), DUCKDB_SCALAR_FUNCTION_SET(VariantExistsFun), DUCKDB_SCALAR_FUNCTION_SET(VariantExtractFun), DUCKDB_SCALAR_FUNCTION_SET(VariantKeysFun), diff --git a/src/duckdb/src/function/scalar/system/aggregate_export.cpp b/src/duckdb/src/function/scalar/system/aggregate_export.cpp index 5b5aca876..d832cad16 100644 --- a/src/duckdb/src/function/scalar/system/aggregate_export.cpp +++ b/src/duckdb/src/function/scalar/system/aggregate_export.cpp @@ -312,8 +312,14 @@ static void DeserializeField(const LogicalType &type, const AggregateStateField } } -static void DeserializeState(const AggregateStateLayout &layout, const Vector &input_vec, idx_t count, - data_ptr_t dest_buffer, ArenaAllocator &allocator) { +static void DeserializeState(const BoundAggregateFunction &aggr, const AggregateStateLayout &layout, + const Vector &input_vec, idx_t count, data_ptr_t dest_buffer, ArenaAllocator &allocator) { + if (aggr.HasImportAggregateStateCallback()) { + // the aggregate explicitly deserializes its own states + AggregateImportInputData import_input(layout, input_vec, dest_buffer, allocator); + aggr.GetImportAggregateStateCallback()(import_input); + return; + } DeserializeField(layout.type, layout.field, input_vec, count, dest_buffer, layout.total_state_size, 0, allocator); } @@ -322,6 +328,36 @@ static void SerializeState(const AggregateStateLayout &layout, Vector &result, i SerializeField(layout.type, layout.field, result, count, addresses, 0); } +static void SerializeState(const BoundAggregateFunction &aggr, optional_ptr bind_data, + const AggregateStateLayout &layout, Vector &states, idx_t count, Vector &result, + ArenaAllocator &allocator) { + if (aggr.HasExportAggregateStateCallback()) { + // the aggregate explicitly serializes its own states + AggregateFinalizeInputData aggr_input_data(aggr, bind_data, allocator); + aggr.GetExportAggregateStateCallback()(states, aggr_input_data, result, count, 0); + return; + } + const data_ptr_t *addresses; + if (states.GetVectorType() == VectorType::CONSTANT_VECTOR) { + addresses = ConstantVector::GetData(states); + } else { + addresses = FlatVector::GetData(states); + } + SerializeState(layout, result, count, addresses); +} + +// destroys the temporary underlying-aggregate states referenced by `states` (no-op if the aggregate has no +// destructor). the combine/finalize paths below initialize underlying states into scratch buffers - without this +// their owned heap (e.g. approx_top_k's hash map, approx_quantile's t-digest) would leak. +static void DestroyExportStates(const BoundAggregateFunction &aggr, optional_ptr bind_data, + Vector &states, idx_t count, ArenaAllocator &allocator) { + if (count == 0 || !aggr.HasStateDestructorCallback()) { + return; + } + AggregateInputData aggr_input_data(aggr, bind_data, allocator); + aggr.GetStateDestructorCallback()(states, aggr_input_data, count); +} + struct CombineState : public FunctionLocalState { //! The state layout, including the segment functions when the state is a linked list AggregateStateLayout layout; @@ -383,10 +419,12 @@ void AggregateStateFinalize(DataChunk &input, ExpressionState &state_p, Vector & state_vec_writer.WriteValue(local_state.state_buffer.get() + i * layout.total_state_size); } - DeserializeState(layout, input.data[0], count, local_state.state_buffer.get(), local_state.allocator); + DeserializeState(bind_data.aggr, layout, input.data[0], count, local_state.state_buffer.get(), + local_state.allocator); AggregateFinalizeInputData aggr_input_data(bind_data.aggr, bind_data.bind_data.get(), local_state.allocator); bind_data.aggr.GetStateFinalizeCallback()(local_state.addresses, aggr_input_data, result, count, 0); + DestroyExportStates(bind_data.aggr, bind_data.bind_data.get(), local_state.addresses, count, local_state.allocator); auto validity = input.data[0].Validity(); for (idx_t i = 0; i < count; i++) { @@ -431,14 +469,31 @@ void AggregateStateCombine(DataChunk &input, ExpressionState &state_p, Vector &r } // Deserialize both inputs — null rows are skipped by LoadOp, keeping the initialized empty state - DeserializeState(layout, input.data[0], count, local_state.state_buffer0.get(), local_state.allocator); - DeserializeState(layout, input.data[1], count, local_state.state_buffer1.get(), local_state.allocator); + DeserializeState(bind_data.aggr, layout, input.data[0], count, local_state.state_buffer0.get(), + local_state.allocator); + DeserializeState(bind_data.aggr, layout, input.data[1], count, local_state.state_buffer1.get(), + local_state.allocator); AggregateInputData aggr_input_data(bind_data.aggr, bind_data.bind_data.get(), local_state.allocator, AggregateCombineType::ALLOW_DESTRUCTIVE); - bind_data.aggr.GetStateCombineCallback()(local_state.addresses0, local_state.addresses1, aggr_input_data, count); - - SerializeState(layout, result, count, FlatVector::GetData(local_state.addresses1)); + // the combine/serialize may throw (e.g. combining approx_top_k states with different k) - always destroy the + // scratch states so their owned heap is not leaked + auto destroy_states = [&]() { + DestroyExportStates(bind_data.aggr, bind_data.bind_data.get(), local_state.addresses0, count, + local_state.allocator); + DestroyExportStates(bind_data.aggr, bind_data.bind_data.get(), local_state.addresses1, count, + local_state.allocator); + }; + try { + bind_data.aggr.GetStateCombineCallback()(local_state.addresses0, local_state.addresses1, aggr_input_data, + count); + SerializeState(bind_data.aggr, bind_data.bind_data.get(), layout, local_state.addresses1, count, result, + local_state.allocator); + } catch (...) { + destroy_states(); + throw; + } + destroy_states(); // Rows where both inputs were NULL produce no meaningful combined state — mark result as NULL for (idx_t i = 0; i < count; i++) { @@ -636,6 +691,14 @@ void CombineAggrStateCombine(Vector &source, Vector &target, AggregateInputData bind_data.aggr.GetStateCombineCallback()(source, target, combine_input, count); } +// destroys combine_aggr's accumulator states by forwarding to the underlying aggregate's destructor (with its own bind +// data) - otherwise states that own heap (e.g. approx_top_k's hash map, approx_quantile's t-digest) would leak +void CombineAggrStateDestroy(Vector &state, AggregateInputData &aggr_input_data, idx_t count) { + auto &bind_data = aggr_input_data.bind_data->Cast(); + AggregateInputData destroy_input(bind_data.aggr, bind_data.bind_data.get(), aggr_input_data.allocator); + bind_data.aggr.GetStateDestructorCallback()(state, destroy_input, count); +} + unique_ptr CombineAggrBind(BindAggregateFunctionInput &input) { auto &context = input.GetClientContext(); auto &function = input.GetBoundFunction(); @@ -647,6 +710,9 @@ unique_ptr CombineAggrBind(BindAggregateFunctionInput &input) { function.SetStateSizeCallback(bind_data->aggr.GetStateSizeCallback()); function.SetStateInitCallback(bind_data->aggr.GetStateInitCallback()); function.SetStateCombineCallback(CombineAggrStateCombine); + if (bind_data->aggr.HasStateDestructorCallback()) { + function.SetStateDestructorCallback(CombineAggrStateDestroy); + } function.SetReturnType(arguments[0]->GetReturnType()); @@ -683,11 +749,13 @@ void CombineAggrUpdate(Vector inputs[], AggregateInputData &aggr_input_data, idx target_data.WriteValue(state_values[i].GetValue()); } - DeserializeState(layout, inputs[0], count, temp_state_buf.get(), aggr_input_data.allocator); + DeserializeState(underlying_aggr, layout, inputs[0], count, temp_state_buf.get(), aggr_input_data.allocator); AggregateInputData combine_input(bind_data.aggr, bind_data.bind_data.get(), aggr_input_data.allocator, AggregateCombineType::ALLOW_DESTRUCTIVE); underlying_aggr.GetStateCombineCallback()(source_vec, target_vec, combine_input, count); + // the target states are the real combine_aggr states (kept); the source states are scratch - destroy them + DestroyExportStates(underlying_aggr, bind_data.bind_data.get(), source_vec, count, aggr_input_data.allocator); } void CombineAggrFinalize(Vector &state, AggregateFinalizeInputData &aggr_input_data, Vector &result, idx_t count, @@ -695,20 +763,11 @@ void CombineAggrFinalize(Vector &state, AggregateFinalizeInputData &aggr_input_d D_ASSERT(offset == 0); auto &bind_data = aggr_input_data.bind_data->Cast(); auto &underlying_aggr = bind_data.aggr; - const data_ptr_t *addresses_ptrs; - if (state.GetVectorType() == VectorType::CONSTANT_VECTOR) { - if (count != 1) { - throw InternalException("Finalize with a constant vector only supported with count of 1"); - } - addresses_ptrs = ConstantVector::GetData(state); - } else { - addresses_ptrs = FlatVector::GetData(state); - } auto layout = GetLayout(underlying_aggr, bind_data.bind_data.get()); result.Flatten(); - SerializeState(layout, result, count, addresses_ptrs); + SerializeState(underlying_aggr, bind_data.bind_data.get(), layout, state, count, result, aggr_input_data.allocator); } // constructs the AGGREGATE_STATE type for the given bound aggregate function @@ -858,7 +917,10 @@ void ToAggregateStateFunction(DataChunk &input, ExpressionState &state, Vector & void ExportAggregateFunction::SetStateExport(BoundAggregateExpression &aggregate, LogicalType state_layout) { auto &bound_function = aggregate.FunctionMutable(); - bound_function.SetStateFinalizeCallback(ExportAggregateFinalize); + // functions with an explicit export callback use it as the finalize; others use the field-based serialization + bound_function.SetStateFinalizeCallback(bound_function.HasExportAggregateStateCallback() + ? bound_function.GetExportAggregateStateCallback() + : ExportAggregateFinalize); // statistics propagation is no longer correct post bound_function.SetStatisticsCallback(nullptr); bound_function.SetReturnType(state_layout); @@ -874,6 +936,11 @@ ExportAggregateFunction::Bind(unique_ptr child_aggrega if (!bound_function.HasStateCombineCallback()) { throw BinderException("Cannot use EXPORT_STATE for non-combinable function %s", bound_function.GetName()); } + if (bound_function.HasExportAggregateStateCallback() != bound_function.HasImportAggregateStateCallback()) { + throw InternalException("Aggregate function \"%s\" must define either both or neither of the " + "export_aggregate_state/import_aggregate_state callbacks", + bound_function.GetName()); + } if (!bound_function.HasGetStateTypeCallback()) { throw NotImplementedException( "Aggregate function \"%s\" does not have a state type callback defined - cannot export state", diff --git a/src/duckdb/src/function/scalar/variant/variant_comparator.cpp b/src/duckdb/src/function/scalar/variant/variant_comparator.cpp new file mode 100644 index 000000000..c3661c575 --- /dev/null +++ b/src/duckdb/src/function/scalar/variant/variant_comparator.cpp @@ -0,0 +1,529 @@ +#include "duckdb/function/scalar/variant_functions.hpp" +#include "duckdb/common/radix.hpp" +#include "duckdb/common/helper.hpp" +#include "duckdb/common/types/datetime.hpp" +#include "duckdb/common/types/interval.hpp" +#include "duckdb/common/types/uhugeint.hpp" +#include "duckdb/common/types/bignum.hpp" +#include "duckdb/common/types/bit.hpp" +#include "duckdb/common/types/variant.hpp" +#include "duckdb/common/enum_util.hpp" +#include "duckdb/common/vector/flat_vector.hpp" +#include "duckdb/common/types/variant_iterator.hpp" + +namespace duckdb { + +namespace { + +//===--------------------------------------------------------------------===// +// Variant Comparator Encoding +//===--------------------------------------------------------------------===// +// VARIANT is physically a STRUCT (keys/children/values/data), but ordering it by that physical +// representation does not match the logical ordering of the values it stores. Instead we traverse +// the variant and emit a sort key per value as [type-rank][value]. Sorting is performed by type +// first and value second. Object keys are processed in string-sorted order so that objects that +// only differ in key order (e.g. {'b': 10, 'a': 1} and {'a': 1, 'b': 10}) compare equal. +// +// Types are grouped into "ranks" the way a semi-structured system (e.g. Snowflake) groups them: +// all integers, decimals and bignums fold into a single NUMBER rank (compared by numeric value) and +// FLOAT/DOUBLE fold into a single REAL rank. Everything else keeps its own rank. +// +// The produced key is byte-comparable and self-contained - it is only ever compared against other +// variant_comparator outputs, so the byte constants below only need to be internally consistent +// (they mirror the conventions used by create_sort_key). + +//! byte constants for the comparator encoding +constexpr data_t STRING_DELIMITER = 0; +constexpr data_t LIST_DELIMITER = 0; +constexpr data_t BLOB_ESCAPE_CHARACTER = 1; + +//! BIT/BITSTRING sort-key bytes (mirrors CreateBitStringSortKey in the core_functions bitstring code): +//! each bit is expanded to one byte, kept above the STRING_DELIMITER (0) so a shorter bitstring (a +//! prefix) sorts first, while preserving 0 < 1 +constexpr data_t BIT_SORT_KEY_ZERO = 2; +constexpr data_t BIT_SORT_KEY_ONE = 3; + +//! nanoseconds-per-unit scale factors used to fold the DATE / TIME / TIMESTAMP precisions into a +//! common unit (nanoseconds) so that values of different precision compare by their actual instant +constexpr int64_t NANOS_PER_MICRO = 1000; +constexpr int64_t NANOS_PER_MILLI = 1000000; +constexpr int64_t NANOS_PER_SEC = 1000000000; +constexpr int64_t NANOS_PER_DAY = 86400LL * NANOS_PER_SEC; + +//! The sort-key rank of a variant value - determines the cross-type ordering (type-first). +//! Ranks start at 1 so that every rank is strictly greater than the LIST/STRING delimiter (0); this +//! guarantees a shorter list/object sorts before a longer one (the delimiter is smaller than any +//! following element's rank byte) and that an element can never be confused with an end-of-list marker. +enum class VariantSortRank : data_t { + BOOLEAN = 1, + NUMBER, + REAL, + VARCHAR, + BLOB, + UUID, + TIMESTAMP, // DATE and all non-tz TIMESTAMP precisions, compared as nanoseconds since the epoch + TIMESTAMP_TZ, // TIMESTAMP WITH TIME ZONE (all precisions), compared as nanoseconds since the epoch (UTC) + TIME, // TIME (all precisions), compared as nanoseconds since midnight + TIME_TZ, // TIME WITH TIME ZONE + INTERVAL, + GEOMETRY, + BITSTRING, + ARRAY, + OBJECT, + //! VARIANT_NULL ranks last so a nested NULL element orders after all non-null values, matching the + //! native "NULLS LAST" ordering of nested NULLs under ASC. For DESC the ORDER BY operator reverses + //! the (always-ascending) comparator key, which then places nested NULLs first - exactly as native + //! nested ordering does (where the null position within nested types follows ASC/DESC). + NULL_VALUE +}; + +data_t GetVariantTypeRank(VariantLogicalType type_id) { + switch (type_id) { + case VariantLogicalType::VARIANT_NULL: + return static_cast(VariantSortRank::NULL_VALUE); + case VariantLogicalType::BOOL_TRUE: + case VariantLogicalType::BOOL_FALSE: + return static_cast(VariantSortRank::BOOLEAN); + // all integers, decimals and bignums fold into a single NUMBER rank (compared by numeric value) + case VariantLogicalType::INT8: + case VariantLogicalType::INT16: + case VariantLogicalType::INT32: + case VariantLogicalType::INT64: + case VariantLogicalType::INT128: + case VariantLogicalType::UINT8: + case VariantLogicalType::UINT16: + case VariantLogicalType::UINT32: + case VariantLogicalType::UINT64: + case VariantLogicalType::UINT128: + case VariantLogicalType::DECIMAL: + case VariantLogicalType::BIGNUM: + return static_cast(VariantSortRank::NUMBER); + // FLOAT and DOUBLE fold into a single REAL rank + case VariantLogicalType::FLOAT: + case VariantLogicalType::DOUBLE: + return static_cast(VariantSortRank::REAL); + case VariantLogicalType::VARCHAR: + return static_cast(VariantSortRank::VARCHAR); + case VariantLogicalType::BLOB: + return static_cast(VariantSortRank::BLOB); + case VariantLogicalType::UUID: + return static_cast(VariantSortRank::UUID); + // DATE folds into the TIMESTAMP rank (a date compares as midnight of that day), together with all + // non-tz TIMESTAMP precisions (SEC / MILIS / MICROS / NANOS) + case VariantLogicalType::DATE: + case VariantLogicalType::TIMESTAMP_SEC: + case VariantLogicalType::TIMESTAMP_MILIS: + case VariantLogicalType::TIMESTAMP_MICROS: + case VariantLogicalType::TIMESTAMP_NANOS: + return static_cast(VariantSortRank::TIMESTAMP); + // TIMESTAMP WITH TIME ZONE precisions fold into a single TIMESTAMP_TZ rank + case VariantLogicalType::TIMESTAMP_MICROS_TZ: + case VariantLogicalType::TIMESTAMP_NANOS_TZ: + return static_cast(VariantSortRank::TIMESTAMP_TZ); + // TIME precisions fold into a single TIME rank + case VariantLogicalType::TIME_MICROS: + case VariantLogicalType::TIME_NANOS: + return static_cast(VariantSortRank::TIME); + case VariantLogicalType::TIME_MICROS_TZ: + return static_cast(VariantSortRank::TIME_TZ); + case VariantLogicalType::INTERVAL: + return static_cast(VariantSortRank::INTERVAL); + case VariantLogicalType::GEOMETRY: + return static_cast(VariantSortRank::GEOMETRY); + case VariantLogicalType::BITSTRING: + return static_cast(VariantSortRank::BITSTRING); + case VariantLogicalType::ARRAY: + return static_cast(VariantSortRank::ARRAY); + case VariantLogicalType::OBJECT: + return static_cast(VariantSortRank::OBJECT); + default: + throw NotImplementedException("Variant type %s is not supported in variant_comparator", + EnumUtil::ToString(type_id)); + } +} + +//===--------------------------------------------------------------------===// +// NUMBER rank encoding +//===--------------------------------------------------------------------===// +// All integers, decimals and bignums fold into a single NUMBER rank and are compared by numeric +// value. To make values of different scales/widths comparable (and equal when numerically equal), +// each value is reduced to an order-preserving "decimal-scientific" form: +// +// value = sign * 0.d1 d2 ... dk * 10^adjexp (d1 != 0, trailing zeros stripped) +// +// and encoded as [class][adjexp][digits][terminator], where everything after the class byte is +// complemented for negatives. This makes e.g. 1::TINYINT, 1::BIGINT, 1.00::DECIMAL and 1::BIGNUM all +// encode identically, and orders them numerically (so -100.5 < 0 < 1.5 < 15 < ...). +struct VariantNumberKey { + //! 0 = negative, 1 = zero, 2 = positive (so negatives sort before zero before positives) + data_t number_class; + int64_t adjusted_exponent; + //! significant decimal digits ('0'..'9'), leading digit non-zero, trailing zeros stripped + string digits; +}; + +//! Build a number key from a sign, the decimal digits of the magnitude (no leading zeros, as e.g. +//! produced by Uhugeint::ToString) and a base-10 scale (value = magnitude / 10^scale) +VariantNumberKey BuildNumberKey(bool negative, const string &magnitude_digits, int64_t scale) { + VariantNumberKey key; + // find the first non-zero digit (magnitude strings have no leading zeros, but be defensive) + idx_t first = 0; + while (first < magnitude_digits.size() && magnitude_digits[first] == '0') { + first++; + } + if (first == magnitude_digits.size()) { + // the value is zero + key.number_class = 1; + key.adjusted_exponent = 0; + return key; + } + key.number_class = negative ? 0 : 2; + // position of the leading digit: (#digits after leading zeros - 1) - scale + key.adjusted_exponent = static_cast(magnitude_digits.size() - first) - 1 - scale; + // strip trailing zeros (keep at least the leading digit) + idx_t last = magnitude_digits.size(); + while (last > first + 1 && magnitude_digits[last - 1] == '0') { + last--; + } + key.digits = magnitude_digits.substr(first, last - first); + return key; +} + +template +void VariantEncodeNumber(SINK &sink, const VariantNumberKey &key) { + // the class byte is the primary discriminator and is never complemented + sink.Write(key.number_class); + if (key.number_class == 1) { + // zero - the class fully describes the value + return; + } + const bool negative = key.number_class == 0; + // for negatives we complement every payload byte so that larger magnitudes sort earlier + auto emit = [&](data_t b) { + sink.Write(negative ? static_cast(~b) : b); + }; + + // adjusted exponent (order-preserving signed encoding) orders magnitude + data_t exp_buffer[sizeof(int64_t)]; + Radix::EncodeData(exp_buffer, key.adjusted_exponent); + for (idx_t i = 0; i < sizeof(int64_t); i++) { + emit(exp_buffer[i]); + } + // significant digits (1..10 so the 0x00 terminator is smaller than any digit -> prefix sorts first) + for (auto c : key.digits) { + emit(static_cast((c - '0') + 1)); + } + emit(0); // terminator +} + +//! Decompose a 128-bit signed value (hugeint) into (sign, magnitude) +uhugeint_t HugeintMagnitude(hugeint_t value, bool &negative) { + negative = value.upper < 0; + uint64_t hi = static_cast(value.upper); + uint64_t lo = value.lower; + if (negative) { + // two's complement negate to get the magnitude + hi = ~hi; + lo = ~lo; + if (++lo == 0) { + ++hi; + } + } + return uhugeint_t(hi, lo); +} + +//! Decimal-digit string of a 128-bit magnitude. Uses a cheap 64-bit conversion when the value fits in +//! 64 bits, avoiding the slower 128-bit Uhugeint::ToString. +string MagnitudeToString(uhugeint_t magnitude) { + if (magnitude.upper == 0) { + return to_string(magnitude.lower); + } + return Uhugeint::ToString(magnitude); +} + +//! Build a number key from a native (<= 64-bit) integer value, computing the magnitude in 64-bit +//! arithmetic instead of widening every value to uhugeint_t. +template +VariantNumberKey IntegerNumberKey(T value) { + bool negative = false; + auto magnitude = static_cast(value); + if constexpr (std::is_signed::value) { + if (value < 0) { + negative = true; + magnitude = uint64_t(0) - magnitude; // unsigned negate -> |value| (also handles the minimum) + } + } + return BuildNumberKey(negative, to_string(magnitude), 0); +} + +//! Compute the number key for any value in the NUMBER rank (integer, decimal or bignum) +VariantNumberKey VariantGetNumberKey(VariantLogicalType type_id, const VariantIterator &it) { + switch (type_id) { + case VariantLogicalType::DECIMAL: { + auto decimal_data = it.GetDecimal(); + hugeint_t unscaled; + switch (decimal_data.GetPhysicalType()) { + case PhysicalType::INT16: + unscaled = hugeint_t(Load(decimal_data.value_ptr)); + break; + case PhysicalType::INT32: + unscaled = hugeint_t(Load(decimal_data.value_ptr)); + break; + case PhysicalType::INT64: + unscaled = hugeint_t(Load(decimal_data.value_ptr)); + break; + default: + unscaled = Load(decimal_data.value_ptr); + break; + } + bool negative; + auto magnitude = HugeintMagnitude(unscaled, negative); + return BuildNumberKey(negative, MagnitudeToString(magnitude), decimal_data.scale); + } + case VariantLogicalType::BIGNUM: { + auto bignum_blob = it.GetString(); + auto decimal_string = Bignum::BignumToVarchar(bignum_t(bignum_blob)); + bool negative = !decimal_string.empty() && decimal_string[0] == '-'; + idx_t offset = negative ? 1 : 0; + return BuildNumberKey(negative, decimal_string.substr(offset), 0); + } + // integer types - use the native width for the magnitude, only widening to 128-bit when necessary + case VariantLogicalType::INT8: + return IntegerNumberKey(Load(it.GetDataPointer())); + case VariantLogicalType::INT16: + return IntegerNumberKey(Load(it.GetDataPointer())); + case VariantLogicalType::INT32: + return IntegerNumberKey(Load(it.GetDataPointer())); + case VariantLogicalType::INT64: + return IntegerNumberKey(Load(it.GetDataPointer())); + case VariantLogicalType::UINT8: + return IntegerNumberKey(Load(it.GetDataPointer())); + case VariantLogicalType::UINT16: + return IntegerNumberKey(Load(it.GetDataPointer())); + case VariantLogicalType::UINT32: + return IntegerNumberKey(Load(it.GetDataPointer())); + case VariantLogicalType::UINT64: + return IntegerNumberKey(Load(it.GetDataPointer())); + case VariantLogicalType::INT128: { + bool negative; + auto magnitude = HugeintMagnitude(Load(it.GetDataPointer()), negative); + return BuildNumberKey(negative, MagnitudeToString(magnitude), 0); + } + case VariantLogicalType::UINT128: + return BuildNumberKey(false, MagnitudeToString(Load(it.GetDataPointer())), 0); + default: + throw InternalException("VariantGetNumberKey called on non-number type %s", EnumUtil::ToString(type_id)); + } +} + +//! Sink that appends the encoded bytes of a variant value to a (growable) buffer +struct VariantComparatorBufferSink { + explicit VariantComparatorBufferSink(string &buffer) : buffer(buffer) { + } + + string &buffer; + + inline void Write(data_t b) { + buffer.push_back(static_cast(b)); + } + inline void WriteBytes(const_data_ptr_t src, idx_t count) { + buffer.append(const_char_ptr_cast(src), count); + } +}; + +template +void VariantEncodeFixed(SINK &sink, T value) { + data_t buffer[sizeof(T)]; + Radix::EncodeData(buffer, value); + sink.WriteBytes(buffer, sizeof(T)); +} + +template +void VariantEncodeString(SINK &sink, const string_t &str, bool is_varchar) { + auto input_data = const_data_ptr_cast(str.GetData()); + auto input_size = str.GetSize(); + if (is_varchar) { + // VARCHAR is valid UTF-8 (never contains 0xFF) so we use the +1 / null-delimiter scheme + for (idx_t i = 0; i < input_size; i++) { + sink.Write(static_cast(input_data[i] + 1)); + } + } else { + // arbitrary bytes - escape \x00 and \x01 so they cannot collide with the delimiter + for (idx_t i = 0; i < input_size; i++) { + if (input_data[i] <= 1) { + sink.Write(BLOB_ESCAPE_CHARACTER); + } + sink.Write(input_data[i]); + } + } + sink.Write(STRING_DELIMITER); +} + +template +void EncodeVariantValue(const VariantIterator &it, SINK &sink) { + auto type_id = it.GetTypeId(); + // write the type rank - this guarantees values are ordered by type first + sink.Write(GetVariantTypeRank(type_id)); + + switch (type_id) { + case VariantLogicalType::VARIANT_NULL: + // no payload - the rank fully describes the value + break; + case VariantLogicalType::BOOL_TRUE: + sink.Write(1); + break; + case VariantLogicalType::BOOL_FALSE: + sink.Write(0); + break; + // all integers, decimals and bignums fold into the NUMBER rank and compare by numeric value + case VariantLogicalType::INT8: + case VariantLogicalType::INT16: + case VariantLogicalType::INT32: + case VariantLogicalType::INT64: + case VariantLogicalType::INT128: + case VariantLogicalType::UINT8: + case VariantLogicalType::UINT16: + case VariantLogicalType::UINT32: + case VariantLogicalType::UINT64: + case VariantLogicalType::UINT128: + case VariantLogicalType::DECIMAL: + case VariantLogicalType::BIGNUM: + VariantEncodeNumber(sink, VariantGetNumberKey(type_id, it)); + break; + case VariantLogicalType::FLOAT: + // fold FLOAT into the REAL rank by widening to double (lossless) + VariantEncodeFixed(sink, static_cast(it.GetData())); + break; + case VariantLogicalType::DOUBLE: + VariantEncodeFixed(sink, it.GetData()); + break; + case VariantLogicalType::UUID: + VariantEncodeFixed(sink, it.GetData()); + break; + // DATE and all (non-tz) TIMESTAMP precisions fold into the TIMESTAMP rank and compare as the number + // of nanoseconds since the epoch (DATE compares as midnight of that day, TIMESTAMP WITH TIME ZONE + // as its UTC instant). int128 is used so that the conversion to nanoseconds is lossless across the + // full range of each type. The type rank already separates the tz / non-tz groups. + case VariantLogicalType::DATE: + VariantEncodeFixed(sink, hugeint_t(it.GetData()) * hugeint_t(NANOS_PER_DAY)); + break; + case VariantLogicalType::TIMESTAMP_SEC: + VariantEncodeFixed(sink, hugeint_t(it.GetData()) * hugeint_t(NANOS_PER_SEC)); + break; + case VariantLogicalType::TIMESTAMP_MILIS: + VariantEncodeFixed(sink, hugeint_t(it.GetData()) * hugeint_t(NANOS_PER_MILLI)); + break; + case VariantLogicalType::TIMESTAMP_MICROS: + case VariantLogicalType::TIMESTAMP_MICROS_TZ: + VariantEncodeFixed(sink, hugeint_t(it.GetData()) * hugeint_t(NANOS_PER_MICRO)); + break; + case VariantLogicalType::TIMESTAMP_NANOS: + case VariantLogicalType::TIMESTAMP_NANOS_TZ: + VariantEncodeFixed(sink, hugeint_t(it.GetData())); + break; + // TIME precisions fold into the TIME rank and compare as nanoseconds since midnight (fits in int64) + case VariantLogicalType::TIME_MICROS: + VariantEncodeFixed(sink, it.GetData() * NANOS_PER_MICRO); + break; + case VariantLogicalType::TIME_NANOS: + VariantEncodeFixed(sink, it.GetData()); + break; + case VariantLogicalType::TIME_MICROS_TZ: + // TIME WITH TIME ZONE requires a dedicated byte-comparable transform + VariantEncodeFixed(sink, it.GetData().sort_key()); + break; + case VariantLogicalType::INTERVAL: + // normalize the interval so that equal intervals encode identically + VariantEncodeFixed(sink, it.GetData().Normalize()); + break; + case VariantLogicalType::VARCHAR: + VariantEncodeString(sink, it.GetString(), true); + break; + case VariantLogicalType::BLOB: + case VariantLogicalType::GEOMETRY: + VariantEncodeString(sink, it.GetString(), false); + break; + case VariantLogicalType::BITSTRING: { + // BIT is ordered by its logical bit sequence, not by its raw bytes - expand each bit to a sort + // key byte (matching the bitstring_byte_comparable collation), terminated so a shorter bitstring + // (a prefix) sorts before a longer one + auto bitstring = it.GetString(); + auto bit_length = Bit::BitLength(bitstring); + for (idx_t bit_idx = 0; bit_idx < bit_length; bit_idx++) { + sink.Write(Bit::GetBit(bitstring, bit_idx) ? BIT_SORT_KEY_ONE : BIT_SORT_KEY_ZERO); + } + sink.Write(STRING_DELIMITER); + break; + } + case VariantLogicalType::ARRAY: { + for (auto child : it.GetArrayChildren()) { + EncodeVariantValue(child, sink); + } + sink.Write(LIST_DELIMITER); + break; + } + case VariantLogicalType::OBJECT: { + // process the children in string-sorted key order so that objects that only differ in key order + // compare equal + for (auto &entry : it.GetObjectChildren(VariantIterationOrder::LEXICOGRAPHIC)) { + VariantEncodeString(sink, entry.key, true); + EncodeVariantValue(entry.value, sink); + } + sink.Write(LIST_DELIMITER); + break; + } + default: + throw NotImplementedException("Variant type %s is not supported in variant_comparator", + EnumUtil::ToString(type_id)); + } +} + +//! Encodes the *logical* value of a VARIANT into a byte-comparable sort key. VARIANT is physically a +//! STRUCT, but the generic create_sort_key encodes that physical representation (which is reversible, +//! so it can be used by aggregates to store/decode values). For comparison / ordering we instead +//! encode the *logical* value of the variant - this encoding is intentionally not reversible (e.g. +//! all integer widths fold together). NULLs are propagated into the result validity. +void CreateVariantComparator(const Vector &input, idx_t count, Vector &result) { + VariantIteratorState variant(input); + + result.SetVectorType(VectorType::FLAT_VECTOR); + auto writer = FlatVector::Writer(result, count); + + //! reused growable buffer - the key is encoded once and then copied into the result vector + string buffer; + for (idx_t r = 0; r < count; r++) { + auto root = variant.Root(r); + // a VARIANT is only NULL at the root via a genuine SQL NULL (never a VARIANT_NULL value) + if (root.IsNull()) { + // propagate NULL so that NULL = NULL stays NULL and ORDER BY ... NULLS FIRST/LAST is honored + writer.WriteNull(); + continue; + } + // encode the key once into the (reused) buffer, then hand it to the writer (which copies it). + // NULLs are handled via validity above, so no NULL/validity prefix byte is needed in the key. + buffer.clear(); + VariantComparatorBufferSink sink(buffer); + EncodeVariantValue(root, sink); + writer.WriteValue(string_t(buffer.data(), NumericCast(buffer.size()))); + } +} + +// variant_comparator produces a binary, byte-comparable sort key for a VARIANT value. +// It is used as a collation for VARIANT so that comparison / ordering / grouping all operate on the +// logical ordering of variant values (see EncodeVariantValue above). +// Unlike the create_sort_key scalar (which uses SPECIAL_HANDLING and encodes NULLs into the key), this +// function propagates NULL - this is required so that NULL = NULL stays NULL and ORDER BY ... NULLS +// FIRST/LAST is handled by the surrounding operator instead of being baked into the key. +void VariantComparatorFunction(DataChunk &input, ExpressionState &state, Vector &result) { + D_ASSERT(input.ColumnCount() == 1); + CreateVariantComparator(input.data[0], input.size(), result); +} + +} // namespace + +ScalarFunction VariantComparatorFun::GetFunction() { + auto variant_type = LogicalType::VARIANT(); + return ScalarFunction("variant_comparator", {variant_type}, LogicalType::BLOB, VariantComparatorFunction); +} + +} // namespace duckdb diff --git a/src/duckdb/src/function/scalar/variant/variant_extract.cpp b/src/duckdb/src/function/scalar/variant/variant_extract.cpp index 99f76497c..380512963 100644 --- a/src/duckdb/src/function/scalar/variant/variant_extract.cpp +++ b/src/duckdb/src/function/scalar/variant/variant_extract.cpp @@ -110,6 +110,10 @@ static bool TryShreddedExtractRecursive(const Vector &input, const vectorsecond.type != type) { throw NotImplementedException("Unimplemented type for test_vector_types %s", type.ToString()); } result.push_back(entry->second.min_value); diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp index f273972f6..a81baf279 100644 --- a/src/duckdb/src/function/table/version/pragma_version.cpp +++ b/src/duckdb/src/function/table/version/pragma_version.cpp @@ -1,5 +1,5 @@ #ifndef DUCKDB_PATCH_VERSION -#define DUCKDB_PATCH_VERSION "0-dev8815" +#define DUCKDB_PATCH_VERSION "0-dev8946" #endif #ifndef DUCKDB_MINOR_VERSION #define DUCKDB_MINOR_VERSION 6 @@ -8,10 +8,10 @@ #define DUCKDB_MAJOR_VERSION 1 #endif #ifndef DUCKDB_VERSION -#define DUCKDB_VERSION "v1.6.0-dev8815" +#define DUCKDB_VERSION "v1.6.0-dev8946" #endif #ifndef DUCKDB_SOURCE_ID -#define DUCKDB_SOURCE_ID "68d73b7c3a" +#define DUCKDB_SOURCE_ID "18b593788d" #endif #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database.hpp" diff --git a/src/duckdb/src/include/duckdb/catalog/default/default_functions.hpp b/src/duckdb/src/include/duckdb/catalog/default/default_functions.hpp index 6aa04a380..0cc801964 100644 --- a/src/duckdb/src/include/duckdb/catalog/default/default_functions.hpp +++ b/src/duckdb/src/include/duckdb/catalog/default/default_functions.hpp @@ -10,6 +10,7 @@ #include "duckdb/catalog/default/default_generator.hpp" #include "duckdb/parser/parsed_data/create_macro_info.hpp" +#include "duckdb/parser/parser_options.hpp" namespace duckdb { class SchemaCatalogEntry; @@ -27,6 +28,9 @@ class DefaultFunctionGenerator : public DefaultGenerator { SchemaCatalogEntry &schema; DUCKDB_API static unique_ptr CreateInternalMacroInfo(const DefaultMacro &default_macro); + //! Overload taking ParserOptions, so the caller's ParserCache is reused instead of rebuilt per macro. + DUCKDB_API static unique_ptr CreateInternalMacroInfo(const DefaultMacro &default_macro, + ParserOptions options); public: unique_ptr CreateDefaultEntry(ClientContext &context, const Identifier &entry_name) override; diff --git a/src/duckdb/src/include/duckdb/catalog/default/default_table_functions.hpp b/src/duckdb/src/include/duckdb/catalog/default/default_table_functions.hpp index 2901a4684..252981ae7 100644 --- a/src/duckdb/src/include/duckdb/catalog/default/default_table_functions.hpp +++ b/src/duckdb/src/include/duckdb/catalog/default/default_table_functions.hpp @@ -10,6 +10,7 @@ #include "duckdb/catalog/default/default_generator.hpp" #include "duckdb/parser/parsed_data/create_macro_info.hpp" +#include "duckdb/parser/parser_options.hpp" namespace duckdb { class SchemaCatalogEntry; @@ -38,6 +39,9 @@ class DefaultTableFunctionGenerator : public DefaultGenerator { vector GetDefaultEntries() override; static unique_ptr CreateTableMacroInfo(const DefaultTableMacro &default_macro); + //! Overload taking ParserOptions, so the caller's ParserCache is reused instead of rebuilt per macro. + static unique_ptr CreateTableMacroInfo(const DefaultTableMacro &default_macro, + ParserOptions options); private: static unique_ptr CreateInternalTableMacroInfo(const DefaultTableMacro &default_macro, diff --git a/src/duckdb/src/include/duckdb/common/enum_util.hpp b/src/duckdb/src/include/duckdb/common/enum_util.hpp index 549cd9347..d2e0e6b01 100644 --- a/src/duckdb/src/include/duckdb/common/enum_util.hpp +++ b/src/duckdb/src/include/duckdb/common/enum_util.hpp @@ -166,6 +166,8 @@ enum class DatePartSpecifier : uint8_t; enum class DebugInitialize : uint8_t; +enum class DebugOrderVerification : uint8_t; + enum class DebugStatementVerification : uint8_t; enum class DebugVectorVerification : uint8_t; @@ -760,6 +762,9 @@ const char* EnumUtil::ToChars(DatePartSpecifier value); template<> const char* EnumUtil::ToChars(DebugInitialize value); +template<> +const char* EnumUtil::ToChars(DebugOrderVerification value); + template<> const char* EnumUtil::ToChars(DebugStatementVerification value); @@ -1550,6 +1555,9 @@ DatePartSpecifier EnumUtil::FromString(const char *value); template<> DebugInitialize EnumUtil::FromString(const char *value); +template<> +DebugOrderVerification EnumUtil::FromString(const char *value); + template<> DebugStatementVerification EnumUtil::FromString(const char *value); diff --git a/src/duckdb/src/include/duckdb/common/enums/debug_order_verification.hpp b/src/duckdb/src/include/duckdb/common/enums/debug_order_verification.hpp new file mode 100644 index 000000000..368083e0b --- /dev/null +++ b/src/duckdb/src/include/duckdb/common/enums/debug_order_verification.hpp @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/common/enums/debug_order_verification.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/constants.hpp" + +namespace duckdb { + +//! How ORDER BY results should be verified (debug setting) +enum class DebugOrderVerification : uint8_t { + //! No verification (default) - ORDER BY is evaluated normally + NONE = 0, + //! Replace ORDER BY x, y, ... with ORDER BY create_sort_key(x, y, ...) to verify the sort key + //! produces the same ordering as the regular comparison + CREATE_SORT_KEY = 1, + //! Cast every ORDER BY expression to VARIANT (ORDER BY x::VARIANT, y::VARIANT, ...) to verify that + //! the VARIANT comparator produces the same ordering as the regular comparison + VARIANT = 2 +}; + +} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/common/multi_file/base_file_reader.hpp b/src/duckdb/src/include/duckdb/common/multi_file/base_file_reader.hpp index a03e7d9b3..23d8bc6fb 100644 --- a/src/duckdb/src/include/duckdb/common/multi_file/base_file_reader.hpp +++ b/src/duckdb/src/include/duckdb/common/multi_file/base_file_reader.hpp @@ -24,6 +24,18 @@ class BaseUnionData; struct GlobalTableFunctionState; struct LocalTableFunctionState; +struct BaseFileReaderExpression { +public: + BaseFileReaderExpression(unique_ptr expr, vector indexes) + : expression(std::move(expr)), column_indexes(std::move(indexes)) { + } + +public: + unique_ptr expression; + //! The column index(es) referenced by the expression + vector column_indexes; +}; + //! Parent class of single-file readers - this must be inherited from for readers implementing the MultiFileReader //! interface class BaseFileReader : public enable_shared_from_this { @@ -46,7 +58,7 @@ class BaseFileReader : public enable_shared_from_this { unique_ptr filters; //! Expression to execute for a given column (BEFORE executing the filter) //! NOTE: this is only set when we have filters - it can be ignored for readers that don't have filter pushdown - unordered_map> expression_map; + unordered_map expression_map; //! The final types for various expressions - this is ONLY used if UseCastMap() is explicitly enabled unordered_map cast_map; diff --git a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp index 3f0668c5f..6c021a29c 100644 --- a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp +++ b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp @@ -415,7 +415,8 @@ class MultiFileFunction : public TableFunction { if (cast_entry != reader.cast_map.end()) { intermediate_chunk_types.push_back(cast_entry->second); } else if (expr_entry != reader.expression_map.end()) { - intermediate_chunk_types.push_back(expr_entry->second->GetReturnType()); + auto &expression = expr_entry->second.expression; + intermediate_chunk_types.push_back(expression->GetReturnType()); } else if (local_id.IsRowIdColumn()) { //! FIXME: should this generically check for all virtual columns?? intermediate_chunk_types.push_back(LogicalType::ROW_TYPE); @@ -677,14 +678,16 @@ class MultiFileFunction : public TableFunction { do { auto &scan_chunk = data.scan_chunk; - scan_chunk.Reset(); + if (data.scan_blocked) { + data.scan_blocked = false; + } else { + scan_chunk.Reset(); + } auto res = data.reader->Scan(context, *gstate.global_state, *data.local_state, scan_chunk); if (res.GetResultType() == AsyncResultType::BLOCKED) { - if (scan_chunk.size() != 0) { - throw InternalException("Unexpected behaviour from Scan, no rows should be returned"); - } + data.scan_blocked = true; switch (data_p.results_execution_mode) { case AsyncResultsExecutionMode::TASK_EXECUTOR: data_p.async_result = std::move(res); @@ -694,7 +697,7 @@ class MultiFileFunction : public TableFunction { if (res.GetResultType() != AsyncResultType::HAVE_MORE_OUTPUT) { throw InternalException("Unexpected behaviour from ExecuteTasksSynchronously"); } - // scan_chunk.size() is 0, see check above, and result is HAVE_MORE_OUTPUT, we need to loop again + // no completed output yet, loop again to resume the Scan continue; } } diff --git a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_reader.hpp b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_reader.hpp index b159073eb..36c5f505b 100644 --- a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_reader.hpp +++ b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_reader.hpp @@ -27,6 +27,45 @@ class DataChunk; enum class ReaderInitializeType { INITIALIZED, SKIP_READING_FILE }; +struct MultiFileReaderVirtualColumnBinding { +public: + enum class VirtualColumnBindingType : uint8_t { COLUMN_REFERENCE, EXPRESSION, CONSTANT }; + +public: + explicit MultiFileReaderVirtualColumnBinding(Value constant) + : type(VirtualColumnBindingType::CONSTANT), constant(std::move(constant)) { + } + MultiFileReaderVirtualColumnBinding(unique_ptr &&expr, vector &&virtual_column_ids) + : type(VirtualColumnBindingType::EXPRESSION), expression(std::move(expr)), + local_virtual_column_ids(std::move(virtual_column_ids)) { + } + explicit MultiFileReaderVirtualColumnBinding(const MultiFileColumnDefinition &column_ref) + : type(VirtualColumnBindingType::COLUMN_REFERENCE), global_column_reference(column_ref) { + } + +public: + VirtualColumnBindingType GetType() const { + return type; + } + +public: + VirtualColumnBindingType type; + //! CONSTANT state: + //! The constant value to replace the virtual column with + Value constant; + + //! EXPRESSION state: + //! The expression to replace the virtual column with + unique_ptr expression; + //! The column id(s) to provide to the reader to produce the input(s) for the 'expression' + vector local_virtual_column_ids; + vector local_column_ids; + + //! COLUMN_REFERENCE state: + //! The column reference to replace the virtual column with + optional_ptr global_column_reference; +}; + //! The MultiFileReader class provides a set of helper methods to handle scanning from multiple files struct MultiFileReader { public: @@ -186,11 +225,10 @@ struct MultiFileReader { idx_t column_id, const LogicalType &type); //! Gets an expression to evaluate the given virtual column - DUCKDB_API virtual unique_ptr + DUCKDB_API virtual MultiFileReaderVirtualColumnBinding GetVirtualColumnExpression(ClientContext &context, MultiFileReaderData &reader_data, - const vector &local_columns, idx_t &column_id, - const LogicalType &type, MultiFileLocalIndex local_index, - optional_ptr &global_column_reference); + const vector &local_columns, const idx_t column_id, + const LogicalType &type, MultiFileLocalIndex local_index); DUCKDB_API virtual unique_ptr Copy() const; diff --git a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_states.hpp b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_states.hpp index fb8d0d995..58736f3e4 100644 --- a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_states.hpp +++ b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_states.hpp @@ -198,6 +198,8 @@ struct MultiFileLocalState : public LocalTableFunctionState { unique_ptr local_state; //! The chunk written to by the reader, handed to FinalizeChunk to transform to the global schema DataChunk scan_chunk; + //! Whether the last Scan call returned BLOCKED + bool scan_blocked = false; //! The executor to transform scan_chunk into the final result with FinalizeChunk ExpressionExecutor executor; //! Number of rows scanned by this thread (for profiling) diff --git a/src/duckdb/src/include/duckdb/common/types/variant_iterator.hpp b/src/duckdb/src/include/duckdb/common/types/variant_iterator.hpp new file mode 100644 index 000000000..69cac2e30 --- /dev/null +++ b/src/duckdb/src/include/duckdb/common/types/variant_iterator.hpp @@ -0,0 +1,341 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/common/types/variant_iterator.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/types/variant.hpp" +#include "duckdb/common/helper.hpp" +#include "duckdb/common/vector/unified_vector_format.hpp" +#include "duckdb/common/vector/vector_iterator.hpp" +#include "duckdb/common/optional_ptr.hpp" +#include "duckdb/common/vector.hpp" + +namespace duckdb { + +class Vector; +struct UnifiedVectorFormat; + +//! The order in which the children of an OBJECT are iterated +enum class VariantIterationOrder { + //! The order in which the children are physically stored (no sorting) + INTERNAL, + //! Sorted by object key + LEXICOGRAPHIC +}; + +//===--------------------------------------------------------------------===// +// VariantIterator +//===--------------------------------------------------------------------===// +// VariantIterator iterates over the logical values of a VARIANT vector *without* unshredding it. +// +// A VARIANT vector is either stored in its canonical "unshredded" layout: +// STRUCT( +// keys VARCHAR[], +// children STRUCT(key_id UINTEGER, value_id UINTEGER)[], +// values STRUCT(type_id UTINYINT, byte_offset UINTEGER)[], +// data BLOB +// ) +// or as a SHREDDED_VECTOR which combines a typed ("shredded") representation with the leftover +// "unshredded" component for everything that did not fit the shredded schema. +// +// The unshredded component is read through plain VectorIterator<>s over the layout above (see +// UnshreddedVariantLayout) - no RecursiveUnifiedVectorFormat / UnifiedVariantVectorData is involved. +// For a shredded vector the iterator descends into the typed shredded tree directly and only falls +// back to these unshredded iterators for the leftover values, merging the two on the fly. + +//! The VectorIterator<> layout of the canonical (unshredded) VARIANT representation + +//! Reads the canonical (unshredded) VARIANT layout (see UnshreddedVariantLayout) through iterators +class UnshreddedVariantIterator { + using UnshreddedVariantLayout = + VectorStructType, VectorListType>, + VectorListType>, string_t>; + +public: + explicit UnshreddedVariantIterator(const Vector &unshredded); + +public: + //! Whether the top-level row is valid (non-NULL) + bool RowIsValid(idx_t row) const; + //! The type_id of values[value_index] + VariantLogicalType GetTypeId(idx_t row, idx_t value_index) const; + //! The byte_offset of values[value_index] + uint32_t GetByteOffset(idx_t row, idx_t value_index) const; + //! The data BLOB of the row. Returned by reference: the inlined bytes of a returned string_t copy + //! would otherwise dangle once the temporary is destroyed. + const string_t &GetBlob(idx_t row) const; + //! The key string of keys[key_index] + string_t GetKey(idx_t row, idx_t key_index) const; + //! The key_id of children[child_index] + uint32_t GetKeysIndex(idx_t row, idx_t child_index) const; + //! The value_id of children[child_index] + uint32_t GetValuesIndex(idx_t row, idx_t child_index) const; + +private: + VectorIterator data; +}; + +//! A recursive, self-similar view over the typed ("shredded") component of a variant +struct ShreddedVariantIterator { +public: + //! Recursively populates this node (and its children) from a shredded variant Vector + void Build(const Vector &vec); + +public: + //! The unified format of this layer + UnifiedVectorFormat unified; + //! The children layers (list/array element, or struct fields) + vector children; + //! The logical type of this layer + LogicalType logical_type; +}; + +class VariantIterator; + +//! Shared state required to iterate a single VARIANT vector. Owns the vector iterators / flattened +//! vectors that the individual VariantIterator cursors point into - so it must outlive any cursor. +class VariantIteratorState { +public: + explicit VariantIteratorState(const Vector &variant); + +public: + //! Whether the row is a (SQL) NULL variant + bool RowIsValid(idx_t row) const; + //! Returns a cursor pointing at the root value of the given row + VariantIterator Root(idx_t row) const; + +private: + //! The "core": the unshredded component reader (plain vector iterators) + UnshreddedVariantIterator unshredded; + + //! Whether the variant vector is shredded + bool is_shredded = false; + //! The shredded component - the (recursive) view of the root of the shredded tree + ShreddedVariantIterator shredded_format; + + friend class VariantIterator; + friend class VariantArrayIterator; + friend class VariantObjectIterator; +}; + +class VariantArrayIterator; +class VariantObjectIterator; + +//! A lightweight cursor pointing at a single logical VARIANT value. +class VariantIterator { +public: + enum class Kind { + NULL_VALUE, //! a (SQL/variant) NULL value + MISSING, //! an absent value (e.g. a missing object field) + UNSHREDDED, //! a value living in the unshredded component + SHREDDED //! a value living in the shredded (typed) component + }; + +public: + VariantIterator() : state(nullptr), kind(Kind::NULL_VALUE) { + } + +public: + bool IsNull() const { + return kind == Kind::NULL_VALUE; + } + bool IsMissing() const { + return kind == Kind::MISSING; + } + + //! The logical type of the value the cursor points at + VariantLogicalType GetTypeId() const; + + //! Returns the fixed-width primitive payload loaded as T (e.g. GetData()) + template + T GetData() const { + return Load(GetDataPointer()); + } + //! Returns a pointer to the raw payload of a fixed-width primitive value + const_data_ptr_t GetDataPointer() const; + //! Returns the (variable-length) string payload of a VARCHAR/BLOB/BIGNUM/GEOMETRY/BITSTRING value + string_t GetString() const; + //! Returns the decimal payload of a DECIMAL value + VariantDecimalData GetDecimal() const; + + //! Iterates the (key, value) children of an OBJECT value (merging shredded + unshredded) in the + //! requested order. LEXICOGRAPHIC sorts by key (currently materialized + sorted up-front). + VariantObjectIterator GetObjectChildren(VariantIterationOrder order = VariantIterationOrder::INTERNAL) const; + //! Lazily iterates the element values of an ARRAY value + VariantArrayIterator GetArrayChildren() const; + +private: + //! Resolve the shredded node (a "STRUCT(typed_value, [untyped_value_index])" wrapper, or a + //! flattened primitive) at the given index into a concrete cursor + static VariantIterator ResolveShredded(const VariantIteratorState &state, const ShreddedVariantIterator &node, + idx_t index, idx_t row); + + static VariantIterator MakeUnshredded(const VariantIteratorState &state, idx_t row, uint32_t value_index); + static VariantIterator MakeShredded(const VariantIteratorState &state, const ShreddedVariantIterator &content, + idx_t index, idx_t row, uint32_t overlay_value_index); + static VariantIterator MakeNull(const VariantIteratorState &state); + static VariantIterator MakeMissing(const VariantIteratorState &state); + +private: + const VariantIteratorState *state; + Kind kind; + + //! The row this value belongs to (used for the unshredded component / overlay lookups) + idx_t row = 0; + //! UNSHREDDED: 0-based index into the 'values' array of the unshredded component + uint32_t value_index = 0; + + //! SHREDDED: the current layer in the shredded format tree (the resolved 'typed_value', i.e. the + //! object struct, the array list, or the primitive) + optional_ptr shredded_format; + //! SHREDDED: the (logical) index into shredded_format + idx_t shredded_index = 0; + //! SHREDDED OBJECT: 1-based index into the unshredded component holding the leftover fields + //! (0 means there is no leftover object to merge) + uint32_t overlay_value_index = 0; + + friend class VariantIteratorState; + friend class VariantArrayIterator; + friend class VariantObjectIterator; +}; + +//! Lazily iterates the element values of an ARRAY VariantIterator. Random-access: no child cursor is +//! materialized until it is dereferenced. +class VariantArrayIterator { +public: + explicit VariantArrayIterator(const VariantIterator &array); + +public: + idx_t size() const { + return length; + } + VariantIterator operator[](idx_t i) const; + + class Iterator { + public: + Iterator(const VariantArrayIterator &parent, idx_t pos) : parent(parent), pos(pos) { + } + VariantIterator operator*() const { + return parent[pos]; + } + Iterator &operator++() { // NOLINT: match stl API + ++pos; + return *this; + } + bool operator!=(const Iterator &other) const { + return pos != other.pos; + } + + private: + const VariantArrayIterator &parent; + idx_t pos; + }; + Iterator begin() const { // NOLINT: match stl API + return Iterator(*this, 0); + } + Iterator end() const { // NOLINT: match stl API + return Iterator(*this, length); + } + +private: + const VariantIteratorState *state; + idx_t row; + idx_t length; + bool shredded; + //! UNSHREDDED: the 'children' base index; SHREDDED: the list offset of the array's elements + idx_t base; + //! SHREDDED: the element layer of the array + optional_ptr element_node; +}; + +//! A single (key, value) entry of an OBJECT +struct VariantObjectEntry { + string_t key; + VariantIterator value; +}; + +//! Iterates the (key, value) children of an OBJECT VariantIterator, merging the shredded (typed) +//! fields with the leftover unshredded fields. There are two backing modes: +//! - INTERNAL order: lazy forward iteration over the raw entries (skipping missing fields) +//! - LEXICOGRAPHIC order: iterates the materialized + sorted 'ordered_entries' +class VariantObjectIterator { +public: + VariantObjectIterator(const VariantIterator &object, VariantIterationOrder order); + +public: + //! Forward iterator over the object entries. All modes are position-based, so the only difference is + //! how an entry at a position is produced (see Load). + class Iterator { + public: + Iterator(const VariantObjectIterator &parent, idx_t pos) : parent(parent), pos(pos) { + Load(); + } + const VariantObjectEntry &operator*() const { + return current; + } + const VariantObjectEntry *operator->() const { + return ¤t; + } + Iterator &operator++() { // NOLINT: match stl API + ++pos; + Load(); + return *this; + } + bool operator!=(const Iterator &other) const { + return pos != other.pos; + } + + private: + //! Loads the entry at the current position, skipping missing fields + void Load(); + + private: + const VariantObjectIterator &parent; + idx_t pos; + VariantObjectEntry current; + }; + Iterator begin() const { // NOLINT: match stl API + return Iterator(*this, 0); + } + Iterator end() const { // NOLINT: match stl API + return Iterator(*this, EndPos()); + } + +private: + //! Materializes the entry at the given raw position (before missing-field skipping) + VariantObjectEntry RawEntry(idx_t raw_pos) const; + //! The end position in the iteration space + idx_t EndPos() const { + return order == VariantIterationOrder::LEXICOGRAPHIC ? ordered_entries.size() : raw_count; + } + +private: + const VariantIteratorState *state; + idx_t row; + VariantIterationOrder order; + bool shredded; + //! Total number of raw entries (typed fields + leftover fields), before missing-field skipping + idx_t raw_count; + + //! UNSHREDDED: the 'children' base index of the object + idx_t base; + + //! SHREDDED: the object's typed struct layer + its index, the number of typed fields, and the + //! 'children' base index of the leftover (overlay) object + optional_ptr content; + idx_t shredded_index; + idx_t typed_field_count; + idx_t overlay_base; + + //! LEXICOGRAPHIC: the entries materialized and sorted by key up-front + vector ordered_entries; + + friend class Iterator; +}; + +} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/common/types/variant_visitor.hpp b/src/duckdb/src/include/duckdb/common/types/variant_visitor.hpp index ac13446a3..91fb3745d 100644 --- a/src/duckdb/src/include/duckdb/common/types/variant_visitor.hpp +++ b/src/duckdb/src/include/duckdb/common/types/variant_visitor.hpp @@ -33,6 +33,7 @@ class VariantVisitor { template static ReturnType Visit(const UnifiedVariantVectorData &variant, idx_t row, uint32_t values_idx, Args &&...args) { if (!variant.RowIsValid(row)) { + VisitMetadata(VariantLogicalType::VARIANT_NULL, std::forward(args)...); return Visitor::VisitNull(std::forward(args)...); } diff --git a/src/duckdb/src/include/duckdb/common/vector/shredded_vector.hpp b/src/duckdb/src/include/duckdb/common/vector/shredded_vector.hpp index 43e0272ec..852b48afe 100644 --- a/src/duckdb/src/include/duckdb/common/vector/shredded_vector.hpp +++ b/src/duckdb/src/include/duckdb/common/vector/shredded_vector.hpp @@ -33,6 +33,8 @@ class ShreddedVectorBuffer : public VectorBuffer { void SetVectorType(VectorType new_vector_type) override; protected: + buffer_ptr SliceInternal(const LogicalType &type, idx_t offset, idx_t end) override; + buffer_ptr SliceInternal(const LogicalType &type, const SelectionVector &sel, idx_t count) override; buffer_ptr FlattenSliceInternal(const LogicalType &type, const SelectionVector &sel, idx_t count) const override; void VerifyInternal(const LogicalType &type, const SelectionVector &sel, idx_t count) const override; diff --git a/src/duckdb/src/include/duckdb/function/aggregate_function.hpp b/src/duckdb/src/include/duckdb/function/aggregate_function.hpp index 55100de2a..39d81055e 100644 --- a/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +++ b/src/duckdb/src/include/duckdb/function/aggregate_function.hpp @@ -133,6 +133,23 @@ typedef unique_ptr (*aggregate_deserialize_t)(Deserializer &deseri typedef AggregateStateLayout (*aggregate_get_state_type_t)(AggregateLayoutInput &input); +//! Input to the import_aggregate_state callback: deserializes the input_vec.size() exported states from input_vec into +//! dest_buffer (state i at offset i * layout.total_state_size). +struct AggregateImportInputData { + AggregateImportInputData(const AggregateStateLayout &layout, const Vector &input_vec, data_ptr_t dest_buffer, + ArenaAllocator &allocator) + : layout(layout), input_vec(input_vec), dest_buffer(dest_buffer), allocator(allocator) { + } + + const AggregateStateLayout &layout; + const Vector &input_vec; + data_ptr_t dest_buffer; + ArenaAllocator &allocator; +}; + +//! The counterpart of export_aggregate_state, for states a static field layout cannot describe (e.g. a hash map). +typedef void (*aggregate_import_state_t)(AggregateImportInputData &input); + struct AggregateFunctionInfo { DUCKDB_API virtual ~AggregateFunctionInfo(); @@ -255,6 +272,11 @@ class AggregateFunctionCallbacks { aggregate_get_state_type_t get_state_type = nullptr; + //! Explicit state export/import callbacks for states a static field layout cannot describe (e.g. a hash map). + //! Must be set together; export acts as the finalize in state-export mode. + aggregate_finalize_t export_aggregate_state = nullptr; + aggregate_import_state_t import_aggregate_state = nullptr; + bool operator==(const AggregateFunctionCallbacks &rhs) const; bool operator!=(const AggregateFunctionCallbacks &rhs) const; }; @@ -375,6 +397,14 @@ class BaseAggregateFunction { bool HasGetStateTypeCallback() const { return callbacks.get_state_type != nullptr; } aggregate_get_state_type_t GetStateTypeCallback() const { return callbacks.get_state_type; } + + bool HasExportAggregateStateCallback() const { return callbacks.export_aggregate_state != nullptr; } + aggregate_finalize_t GetExportAggregateStateCallback() const { return callbacks.export_aggregate_state; } + void SetExportAggregateStateCallback(aggregate_finalize_t callback) { callbacks.export_aggregate_state = callback; } + + bool HasImportAggregateStateCallback() const { return callbacks.import_aggregate_state != nullptr; } + aggregate_import_state_t GetImportAggregateStateCallback() const { return callbacks.import_aggregate_state; } + void SetImportAggregateStateCallback(aggregate_import_state_t callback) { callbacks.import_aggregate_state = callback; } // clang-format on public: // Extra function info @@ -515,6 +545,17 @@ class AggregateFunction : public BaseAggregateFunction, public SimpleFunction { return *this; } + //! Registers explicit export/import callbacks for states a static field layout cannot describe (e.g. a hash map); + //! get_state_type still supplies the exported logical type. + AggregateFunction &SetStateExportCallbacks(aggregate_get_state_type_t get_state_type_callback, + aggregate_finalize_t export_aggregate_state_callback, + aggregate_import_state_t import_aggregate_state_callback) { + callbacks.get_state_type = get_state_type_callback; + callbacks.export_aggregate_state = export_aggregate_state_callback; + callbacks.import_aggregate_state = import_aggregate_state_callback; + return *this; + } + AggregateFunction &SetClusterCallback(aggregate_cluster_update_t cluster_update) { callbacks.cluster_update = cluster_update; return *this; diff --git a/src/duckdb/src/include/duckdb/function/scalar/variant_functions.hpp b/src/duckdb/src/include/duckdb/function/scalar/variant_functions.hpp index 85dd82ea3..ea308ddc3 100644 --- a/src/duckdb/src/include/duckdb/function/scalar/variant_functions.hpp +++ b/src/duckdb/src/include/duckdb/function/scalar/variant_functions.hpp @@ -55,6 +55,16 @@ struct VariantNormalizeFun { static ScalarFunction GetFunction(); }; +struct VariantComparatorFun { + static constexpr const char *Name = "variant_comparator"; + static constexpr const char *Parameters = "input_variant"; + static constexpr const char *Description = "Returns a binary sort key that orders `input_variant` according to the logical ordering of VARIANT values (used internally for comparison and ordering)."; + static constexpr const char *Example = "variant_comparator({'a': 1, 'b': 2}::VARIANT)"; + static constexpr const char *Categories = "variant"; + + static ScalarFunction GetFunction(); +}; + struct VariantTypeofFun { static constexpr const char *Name = "variant_typeof"; static constexpr const char *Parameters = "input_variant"; diff --git a/src/duckdb/src/include/duckdb/main/config.hpp b/src/duckdb/src/include/duckdb/main/config.hpp index def558ad9..3e359edda 100644 --- a/src/duckdb/src/include/duckdb/main/config.hpp +++ b/src/duckdb/src/include/duckdb/main/config.hpp @@ -38,6 +38,7 @@ #include "duckdb/common/types/type_manager.hpp" #include "duckdb/common/storage_compatibility.hpp" #include "duckdb/common/enums/debug_verification_mode.hpp" +#include "duckdb/common/enums/debug_order_verification.hpp" namespace duckdb { @@ -118,6 +119,8 @@ struct DBConfigOptions { vector extension_directories; //! Debug setting - how to initialize blocks in the storage layer when allocating DebugInitialize debug_initialize = DebugInitialize::NO_INITIALIZE; + //! Debug setting - how to verify ORDER BY results (e.g. by rewriting ORDER BY into create_sort_key) + DebugOrderVerification debug_order_verification = DebugOrderVerification::NONE; //! The set of user-provided options case_insensitive_map_t user_options; //! The set of unrecognized (other) options diff --git a/src/duckdb/src/include/duckdb/main/extension_entries.hpp b/src/duckdb/src/include/duckdb/main/extension_entries.hpp index 51ec375f1..d3ac41792 100644 --- a/src/duckdb/src/include/duckdb/main/extension_entries.hpp +++ b/src/duckdb/src/include/duckdb/main/extension_entries.hpp @@ -1155,6 +1155,7 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = { {"mysql_time_as_time", "mysql_scanner"}, {"mysql_tinyint1_as_boolean", "mysql_scanner"}, {"parquet_metadata_cache", "parquet"}, + {"parquet_prefetch_column_gap", "parquet"}, {"pg_array_as_varchar", "postgres_scanner"}, {"pg_connection_cache", "postgres_scanner"}, {"pg_connection_limit", "postgres_scanner"}, diff --git a/src/duckdb/src/include/duckdb/main/settings.hpp b/src/duckdb/src/include/duckdb/main/settings.hpp index bb269660f..518f46b45 100644 --- a/src/duckdb/src/include/duckdb/main/settings.hpp +++ b/src/duckdb/src/include/duckdb/main/settings.hpp @@ -390,6 +390,17 @@ struct BlockAllocatorMemorySetting { static Value GetSetting(const ClientContext &context); }; +struct CacheLocalFilesSetting { + using RETURN_TYPE = bool; + static constexpr const char *Name = "cache_local_files"; + static constexpr const char *Description = + "Whether the external file cache also caches local files (remote files are always cached)"; + static constexpr const char *InputType = "BOOLEAN"; + static constexpr const char *DefaultValue = "false"; + static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; + static constexpr idx_t SettingIndex = NEXT_SETTING_INDEX(); +}; + struct CatalogErrorMaxSchemasSetting { using RETURN_TYPE = idx_t; static constexpr const char *Name = "catalog_error_max_schemas"; @@ -552,6 +563,17 @@ struct DebugForceNoCrossProductSetting { static constexpr idx_t SettingIndex = NEXT_SETTING_INDEX(); }; +struct DebugOrderVerificationSetting { + using RETURN_TYPE = DebugOrderVerification; + static constexpr const char *Name = "debug_order_verification"; + static constexpr const char *Description = + "DEBUG SETTING: verify ORDER BY results by rewriting the ordering (NONE, CREATE_SORT_KEY or VARIANT)"; + static constexpr const char *InputType = "VARCHAR"; + static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value ¶meter); + static void ResetGlobal(DatabaseInstance *db, DBConfig &config); + static Value GetSetting(const ClientContext &context); +}; + struct DebugPhysicalTableScanExecutionStrategySetting { using RETURN_TYPE = PhysicalTableScanExecutionStrategy; static constexpr const char *Name = "debug_physical_table_scan_execution_strategy"; diff --git a/src/duckdb/src/include/duckdb/parser/peg/ast/cast_arguments.hpp b/src/duckdb/src/include/duckdb/parser/peg/ast/cast_arguments.hpp new file mode 100644 index 000000000..cf9251e53 --- /dev/null +++ b/src/duckdb/src/include/duckdb/parser/peg/ast/cast_arguments.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include "duckdb/common/common.hpp" +#include "duckdb/common/types.hpp" +#include "duckdb/parser/parsed_expression.hpp" + +namespace duckdb { + +struct CastArguments { + unique_ptr expression; + LogicalType type; +}; + +} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/parser/peg/ast/expression_chain.hpp b/src/duckdb/src/include/duckdb/parser/peg/ast/expression_chain.hpp new file mode 100644 index 000000000..db766b8c8 --- /dev/null +++ b/src/duckdb/src/include/duckdb/parser/peg/ast/expression_chain.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include "duckdb/parser/parsed_expression.hpp" + +namespace duckdb { + +struct IsDistinctFromTail { + ExpressionType comparison_type; + unique_ptr expression; +}; + +struct ComparisonExpressionTail { + ExpressionType comparison_type; + vector not_keywords; + unique_ptr expression; +}; + +struct BetweenInLikeOperator { + bool has_not = false; + unique_ptr expression; +}; + +struct ParsedOperator { + string name; + bool is_any_all = false; + bool is_any = true; +}; + +struct OtherOperatorTail { + ParsedOperator op; + unique_ptr expression; +}; + +struct BinaryExpressionTail { + string op; + unique_ptr expression; + optional_idx query_location; +}; + +} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/parser/peg/ast/method_arguments.hpp b/src/duckdb/src/include/duckdb/parser/peg/ast/method_arguments.hpp new file mode 100644 index 000000000..cb733f7c0 --- /dev/null +++ b/src/duckdb/src/include/duckdb/parser/peg/ast/method_arguments.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/result_modifier.hpp" + +namespace duckdb { + +struct MethodArguments { + bool distinct = false; + vector arguments; + vector order_bys; + bool has_ignore_nulls = false; + bool ignore_nulls = false; +}; + +} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/parser/peg/ast/trim_arguments.hpp b/src/duckdb/src/include/duckdb/parser/peg/ast/trim_arguments.hpp new file mode 100644 index 000000000..1116d07c1 --- /dev/null +++ b/src/duckdb/src/include/duckdb/parser/peg/ast/trim_arguments.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "duckdb/common/optional.hpp" +#include "duckdb/parser/parsed_expression.hpp" + +namespace duckdb { + +struct TrimArguments { + optional trim_direction; + vector> expressions; +}; + +} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/parser/peg/inlined_grammar.hpp b/src/duckdb/src/include/duckdb/parser/peg/inlined_grammar.hpp index a98a8ba50..274464d23 100644 --- a/src/duckdb/src/include/duckdb/parser/peg/inlined_grammar.hpp +++ b/src/duckdb/src/include/duckdb/parser/peg/inlined_grammar.hpp @@ -720,7 +720,10 @@ const char INLINED_PEG_GRAMMAR[] = { "CatalogReservedSchemaTableColumnName <- CatalogQualification ReservedSchemaQualification ReservedTableQualification ReservedColumnName\n" "SchemaReservedTableColumnName <- SchemaQualification ReservedTableQualification ReservedColumnName\n" "TableReservedColumnName <- TableQualification ReservedColumnName\n" - "FunctionExpression <- FunctionIdentifier Parens(DistinctOrAll? List(FunctionArgument)? OrderByClause? IgnoreOrRespectNulls?) WithinGroupClause? FilterClause? ExportClause? OverClause?\n" + "FunctionExpression <- FunctionIdentifier FunctionExpressionArguments WithinGroupClause? FilterClause? ExportClause? OverClause?\n" + "FunctionExpressionArguments <- Parens(FunctionExpressionArgumentList)\n" + "FunctionExpressionArgumentList <- DistinctOrAll? FunctionArgumentList? OrderByClause? IgnoreOrRespectNulls?\n" + "FunctionArgumentList <- List(FunctionArgument)\n" "FunctionIdentifier <- CatalogReservedSchemaFunctionName / SchemaReservedFunctionName / FunctionName\n" "CatalogReservedSchemaFunctionName <- CatalogQualification ReservedSchemaQualification? ReservedFunctionName\n" "SchemaReservedFunctionName <- SchemaQualification ReservedFunctionName\n" @@ -729,7 +732,9 @@ const char INLINED_PEG_GRAMMAR[] = { "AllKeyword <- 'ALL'\n" "ExportClause <- 'EXPORT_STATE'\n" "WithinGroupClause <- 'WITHIN' 'GROUP' Parens(OrderByClause)\n" - "FilterClause <- 'FILTER' Parens('WHERE'? Expression)\n" + "FilterClause <- 'FILTER' FilterClauseExpression\n" + "FilterClauseExpression <- Parens(FilterClauseContents)\n" + "FilterClauseContents <- 'WHERE'? Expression\n" "IgnoreOrRespectNulls <- IgnoreNulls / RespectNulls\n" "IgnoreNulls <- 'IGNORE' 'NULLS'\n" "RespectNulls <- 'RESPECT' 'NULLS'\n" @@ -739,33 +744,47 @@ const char INLINED_PEG_GRAMMAR[] = { "NullLiteral <- 'NULL'\n" "TrueLiteral <- 'TRUE'\n" "FalseLiteral <- 'FALSE'\n" - "CastExpression <- CastOrTryCast Parens(Expression 'AS' Type)\n" - "CastOrTryCast <- 'CAST' / 'TRY_CAST'\n" + "CastExpression <- CastOrTryCast Parens(CastArguments)\n" + "CastArguments <- Expression 'AS' Type\n" + "CastOrTryCast <- CastKeyword / TryCastKeyword\n" + "CastKeyword <- 'CAST'\n" + "TryCastKeyword <- 'TRY_CAST'\n" "ColIdDot <- ColId '.'\n" - "StarExpression <- ColIdDot* '*' ExcludeList? ReplaceList? RenameList?\n" - "ExcludeList <- ExcludeOrExcept ExcludeNameList / ExcludeNameSingle\n" + "StarExpression <- StarQualifierList? '*' ExcludeList? ReplaceList? RenameList?\n" + "StarQualifierList <- ColIdDot+\n" + "ExcludeList <- ExcludeOrExcept ExcludeNames\n" "ExcludeOrExcept <- 'EXCLUDE' / 'EXCEPT'\n" + "ExcludeNames <- ExcludeNameList / ExcludeNameSingle\n" "ExcludeNameList <- Parens(List(ExcludeName))\n" "ExcludeNameSingle <- ExcludeName\n" - "ExcludeName <- DottedIdentifier / ColIdOrString\n" + "ExcludeName <- ExcludeDottedName / ExcludeColumnName\n" + "ExcludeDottedName <- DottedIdentifier\n" + "ExcludeColumnName <- ColIdOrString\n" "ReplaceList <- 'REPLACE' ReplaceEntries\n" "ReplaceEntries <- ReplaceEntrySingle / ReplaceEntryList\n" "ReplaceEntrySingle <- ReplaceEntry\n" "ReplaceEntryList <- Parens(List(ReplaceEntry))\n" "ReplaceEntry <- Expression 'AS' ColumnReference\n" - "RenameList <- 'RENAME' (RenameEntryList / SingleRenameEntry)\n" + "RenameList <- 'RENAME' RenameEntries\n" + "RenameEntries <- RenameEntryList / SingleRenameEntry\n" "RenameEntryList <- Parens(List(RenameEntry))\n" "SingleRenameEntry <- RenameEntry\n" "RenameEntry <- ExcludeName 'AS' Identifier\n" - "SubqueryExpression <- 'NOT'? 'EXISTS'? SubqueryReference\n" + "SubqueryExpression <- SubqueryNot? SubqueryExists? SubqueryReference\n" + "SubqueryNot <- 'NOT'\n" + "SubqueryExists <- 'EXISTS'\n" "CaseExpression <- 'CASE' Expression? CaseWhenThen+ CaseElse? 'END'\n" "CaseWhenThen <- 'WHEN' Expression 'THEN' Expression\n" "CaseElse <- 'ELSE' Expression\n" "TypeLiteral <- ColId StringLiteral\n" "IntervalLiteral <- 'INTERVAL' IntervalParameter Interval?\n" - "IntervalParameter <- StringLiteral / NumberLiteral / ParensExpression\n" + "IntervalParameter <- IntervalStringParameter / NumberLiteral / ParensExpression\n" + "IntervalStringParameter <- StringLiteral\n" "FrameClause <- Framing FrameExtent WindowExcludeClause?\n" - "Framing <- 'ROWS' / 'RANGE' / 'GROUPS'\n" + "Framing <- RowsFraming / RangeFraming / GroupsFraming\n" + "RowsFraming <- 'ROWS'\n" + "RangeFraming <- 'RANGE'\n" + "GroupsFraming <- 'GROUPS'\n" "FrameExtent <- BetweenFrameExtent / SingleFrameExtent\n" "SingleFrameExtent <- FrameBound\n" "BetweenFrameExtent <- 'BETWEEN' FrameBound 'AND' FrameBound\n" @@ -773,7 +792,9 @@ const char INLINED_PEG_GRAMMAR[] = { "FrameUnbounded <- 'UNBOUNDED' PrecedingOrFollowing\n" "FrameExpression <- Expression PrecedingOrFollowing\n" "FrameCurrentRow <- 'CURRENT' 'ROW'\n" - "PrecedingOrFollowing <- 'PRECEDING' / 'FOLLOWING'\n" + "PrecedingOrFollowing <- PrecedingFrame / FollowingFrame\n" + "PrecedingFrame <- 'PRECEDING'\n" + "FollowingFrame <- 'FOLLOWING'\n" "WindowExcludeClause <- 'EXCLUDE' WindowExcludeElement\n" "WindowExcludeElement <- ExcludeCurrentRow / ExcludeGroup / ExcludeTies / ExcludeNoOthers\n" "ExcludeCurrentRow <- 'CURRENT' 'ROW'\n" @@ -784,7 +805,8 @@ const char INLINED_PEG_GRAMMAR[] = { "WindowFrame <- ParensIdentifier / WindowFrameDefinition / Identifier\n" "ParensIdentifier <- Parens(Identifier)\n" "WindowFrameDefinition <- WindowFrameNameContentsParens / WindowFrameContentsParens\n" - "WindowFrameNameContentsParens <- Parens(BaseWindowName? WindowFrameContents)\n" + "WindowFrameNameContentsParens <- Parens(WindowFrameNameContents)\n" + "WindowFrameNameContents <- BaseWindowName? WindowFrameContents\n" "WindowFrameContentsParens <- Parens(WindowFrameContents)\n" "WindowFrameContents <- WindowPartition? OrderByClause? FrameClause?\n" "BaseWindowName <- Identifier\n" @@ -799,7 +821,9 @@ const char INLINED_PEG_GRAMMAR[] = { "MapStructExpression <- '{' List(MapStructField)? '}'\n" "MapStructField <- Expression ':' Expression\n" "GroupingExpression <- GroupingOrGroupingId Parens(List(Expression)?)\n" - "GroupingOrGroupingId <- 'GROUPING' / 'GROUPING_ID'\n" + "GroupingOrGroupingId <- GroupingKeyword / GroupingIdKeyword\n" + "GroupingKeyword <- 'GROUPING'\n" + "GroupingIdKeyword <- 'GROUPING_ID'\n" "Parameter <- QuestionMarkNumberedParameter / AnonymousParameter / NumberedParameter / ColLabelParameter\n" "QuestionMarkNumberedParameter <- '?' NumberLiteral\n" "AnonymousParameter <- '?'\n" @@ -837,25 +861,37 @@ const char INLINED_PEG_GRAMMAR[] = { "# LEVEL 1.5\n" "LambdaArrowExpression <- LogicalOrExpression SingleArrowPair*\n" "SingleArrowPair <- '->' LogicalOrExpression\n" - "LogicalOrExpression <- LogicalAndExpression ('OR' LogicalAndExpression)*\n" - "ColDefOrExpr <- ColDefAndExpr ('OR' ColDefAndExpr)*\n" + "LogicalOrExpression <- LogicalAndExpression LogicalOrExpressionTail*\n" + "LogicalOrExpressionTail <- 'OR' LogicalAndExpression\n" + "ColDefOrExpr <- ColDefAndExpr ColDefOrExpressionTail*\n" + "ColDefOrExpressionTail <- 'OR' ColDefAndExpr\n" "# LEVEL 2\n" - "LogicalAndExpression <- LogicalNotExpression ('AND' LogicalNotExpression)*\n" - "ColDefAndExpr <- IsDistinctFromExpression ('AND' IsDistinctFromExpression)*\n" + "LogicalAndExpression <- LogicalNotExpression LogicalAndExpressionTail*\n" + "LogicalAndExpressionTail <- 'AND' LogicalNotExpression\n" + "ColDefAndExpr <- IsDistinctFromExpression ColDefAndExpressionTail*\n" + "ColDefAndExpressionTail <- 'AND' IsDistinctFromExpression\n" "# LEVEL 3\n" - "LogicalNotExpression <- 'NOT'* IsExpression\n" + "LogicalNotExpression <- NotExpression? IsExpression\n" + "NotExpression <- NotKeyword+\n" + "NotKeyword <- 'NOT'\n" "# LEVEL 4\n" "IsExpression <- IsDistinctFromExpression IsTest*\n" "IsTest <- IsLiteral / NotNull / IsNull\n" - "IsLiteral <- 'IS' 'NOT'? (TrueLiteral / FalseLiteral / NullLiteral / UnknownLiteral)\n" + "IsLiteral <- 'IS' 'NOT'? IsLiteralValue\n" + "IsLiteralValue <- TrueLiteral / FalseLiteral / NullLiteral / UnknownLiteral\n" "UnknownLiteral <- 'UNKNOWN'\n" - "NotNull <- ('NOT' 'NULL') / 'NOTNULL'\n" - "IsNull <- 'ISNULL'\n" + "NotNull <- NotNullKeyword / NotNullOperator\n" + "NotNullKeyword <- 'NOT' 'NULL'\n" + "NotNullOperator <- 'NOTNULL'\n" + "IsNull <- IsNullOperator\n" + "IsNullOperator <- 'ISNULL'\n" "# LEVEL 5 (Split because IsDistinctFromExpression allows post expression while IsOperator does not)\n" - "IsDistinctFromExpression <- ComparisonExpression (IsDistinctFromOp ComparisonExpression)*\n" + "IsDistinctFromExpression <- ComparisonExpression IsDistinctFromTail*\n" + "IsDistinctFromTail <- IsDistinctFromOp ComparisonExpression\n" "IsDistinctFromOp <- 'IS' 'NOT'? 'DISTINCT' 'FROM'\n" "# LEVEL 6\n" - "ComparisonExpression <- BetweenInLikeExpression (ComparisonOperator 'NOT'* BetweenInLikeExpression)*\n" + "ComparisonExpression <- BetweenInLikeExpression ComparisonExpressionTail*\n" + "ComparisonExpressionTail <- ComparisonOperator NotExpression? BetweenInLikeExpression\n" "ComparisonOperator <-\n" " OperatorEqual /\n" " OperatorNotEqual /\n" @@ -871,7 +907,8 @@ const char INLINED_PEG_GRAMMAR[] = { "OperatorGreaterThanEquals <- '>='\n" "# LEVEL 7\n" "BetweenInLikeExpression <- OtherOperatorExpression BetweenInLikeOp?\n" - "BetweenInLikeOp <- 'NOT'? (BetweenClause / InClause / LikeClause)\n" + "BetweenInLikeOp <- 'NOT'? BetweenInLikeOpExpression\n" + "BetweenInLikeOpExpression <- BetweenClause / InClause / LikeClause\n" "LikeClause <- LikeVariations OtherOperatorExpression EscapeClause?\n" "EscapeClause <- 'ESCAPE' ComparisonExpression\n" "LikeVariations <- SimilarToToken / ILikeToken / LikeToken / GlobToken / NotILikeOp / NotLikeOp / NotSimilarToOp\n" @@ -883,12 +920,14 @@ const char INLINED_PEG_GRAMMAR[] = { "NotLikeOp <- '!~~'\n" "NotSimilarToOp <- '!~'\n" "InClause <- 'IN' InExpression\n" - "InExpression <- InExpressionList / InSelectStatement / OtherOperatorExpression\n" + "InExpression <- InExpressionList / InSelectStatement / InContainsExpression\n" + "InContainsExpression <- OtherOperatorExpression\n" "InExpressionList <- Parens(List(Expression))\n" "InSelectStatement <- Parens(SelectStatementInternal)\n" "BetweenClause <- 'BETWEEN' OtherOperatorExpression 'AND' OtherOperatorExpression\n" "# LEVEL 8\n" - "OtherOperatorExpression <- BitwiseExpression (OtherOperator BitwiseExpression)*\n" + "OtherOperatorExpression <- BitwiseExpression OtherOperatorTail*\n" + "OtherOperatorTail <- OtherOperator BitwiseExpression\n" "OtherOperator <-\n" " QualifiedOperator / AnyAllOperator / InetOperator / JsonOperator / ListOperator / StringOperator / OperatorLiteral\n" "OperatorLiteral <- Identifier\n" @@ -900,25 +939,32 @@ const char INLINED_PEG_GRAMMAR[] = { "JsonOperator <- '->>'\n" "ListOperator <- '&&' / '@>' / '<@'\n" "StringOperator <- '^@' / '||'\n" - "QualifiedOperator <- 'OPERATOR' Parens(ColIdDot* AnyOp)\n" + "QualifiedOperator <- 'OPERATOR' Parens(QualifiedOperatorContents)\n" + "QualifiedOperatorContents <- ColIdDot* AnyOp\n" "AnyOp <- '!~~*' / '>>=' / '<<=' / '->>' / '!~~' / '~~*' / '~~~' / '!~' / '^@' / '||' / '&&' / '@>' / '<@' / '<=' / '>=' / '<>' / '!=' / '==' / '<<' / '>>' / '//' / '**' / '->' / '~~' / '+' / '-' / '*' / '/' / '%' / '^' / '<' / '>' / '=' / '&' / '|' / '~' / '!'\n" "# LEVEL 9\n" - "BitwiseExpression <- AdditiveExpression (BitOperator AdditiveExpression)*\n" + "BitwiseExpression <- AdditiveExpression BitwiseExpressionTail*\n" + "BitwiseExpressionTail <- BitOperator AdditiveExpression\n" "BitOperator <- '&' / '|' / '<<' / '>>'\n" "# LEVEL 10\n" - "AdditiveExpression <- MultiplicativeExpression (Term MultiplicativeExpression)*\n" + "AdditiveExpression <- MultiplicativeExpression AdditiveExpressionTail*\n" + "AdditiveExpressionTail <- Term MultiplicativeExpression\n" "Term <- '+' / '-'\n" "# LEVEL 11\n" - "MultiplicativeExpression <- ExponentiationExpression (Factor ExponentiationExpression)*\n" + "MultiplicativeExpression <- ExponentiationExpression MultiplicativeExpressionTail*\n" + "MultiplicativeExpressionTail <- Factor ExponentiationExpression\n" "Factor <- '*' / '/' / '//' / '%'\n" "# LEVEL 12\n" - "ExponentiationExpression <- CollateExpression (ExponentOperator CollateExpression)*\n" + "ExponentiationExpression <- CollateExpression ExponentiationExpressionTail*\n" + "ExponentiationExpressionTail <- ExponentOperator CollateExpression\n" "ExponentOperator <- '^' / '**'\n" "# LEVEL 13\n" - "CollateExpression <- AtTimeZoneExpression (CollateOperator AtTimeZoneExpression)*\n" + "CollateExpression <- AtTimeZoneExpression CollateExpressionTail*\n" + "CollateExpressionTail <- CollateOperator AtTimeZoneExpression\n" "CollateOperator <- 'COLLATE'\n" "# LEVEL 14\n" - "AtTimeZoneExpression <- PrefixExpression (AtTimeZoneOperator PrefixExpression)*\n" + "AtTimeZoneExpression <- PrefixExpression AtTimeZoneExpressionTail*\n" + "AtTimeZoneExpressionTail <- AtTimeZoneOperator PrefixExpression\n" "AtTimeZoneOperator <- 'AT' 'TIME' 'ZONE'\n" "# LEVEL 15\n" "PrefixExpression <- PrefixOperator* BaseExpression\n" @@ -927,25 +973,36 @@ const char INLINED_PEG_GRAMMAR[] = { "PlusPrefixOperator <- '+'\n" "TildePrefixOperator <- '~'\n" "# LEVEL 16 (Highest)\n" - "BaseExpression <- SingleExpression Indirection*\n" + "BaseExpression <- SingleExpression IndirectionList?\n" + "IndirectionList <- Indirection+\n" "Indirection <- CastOperator / DotOperator / SliceExpression / PostfixOperator\n" "CastOperator <- '::' Type\n" - "DotOperator <- '.' (MethodExpression / ColLabel)\n" - "MethodExpression <- ColLabel Parens(DistinctOrAll? List(FunctionArgument)? OrderByClause? IgnoreOrRespectNulls?)\n" + "DotOperator <- DotMethodOperator / DotColumnOperator\n" + "DotMethodOperator <- '.' MethodExpression\n" + "DotColumnOperator <- '.' ColLabel\n" + "MethodExpression <- ColLabel MethodExpressionArguments\n" + "MethodExpressionArguments <- Parens(MethodExpressionArgumentList)\n" + "MethodExpressionArgumentList <- DistinctOrAll? MethodFunctionArguments? OrderByClause? IgnoreOrRespectNulls?\n" + "MethodFunctionArguments <- List(FunctionArgument)\n" "SliceExpression <- '[' SliceBound ']'\n" "SliceBound <- Expression? EndSliceBound? StepSliceBound?\n" - "EndSliceBound <- ':' (Expression / '-')?\n" + "EndSliceBound <- ':' EndSliceValue?\n" + "EndSliceValue <- Expression / EndSliceMinus\n" + "EndSliceMinus <- '-'\n" "StepSliceBound <- ':' Expression?\n" "PostfixOperator <- '!'\n" "SpecialFunctionExpression <- CoalesceExpression / UnpackExpression / TryExpression / ColumnsExpression / ExtractExpression / LambdaExpression / NullIfExpression / PositionExpression / RowExpression / SubstringExpression / TrimExpression / OverlayExpression\n" "CoalesceExpression <- 'COALESCE' Parens(List(Expression))\n" "UnpackExpression <- 'UNPACK' Parens(Expression)\n" "TryExpression <- 'TRY' Parens(Expression)\n" - "ColumnsExpression <- '*'? 'COLUMNS' Parens(Expression)\n" - "ExtractExpression <- 'EXTRACT' Parens(ExtractArgument 'FROM' Expression)\n" + "ColumnsExpression <- StarSymbol? 'COLUMNS' Parens(Expression)\n" + "ExtractExpression <- 'EXTRACT' Parens(ExtractArguments)\n" + "ExtractArguments <- ExtractArgument 'FROM' Expression\n" "LambdaExpression <- 'LAMBDA' List(ColIdOrString) ':' Expression\n" - "NullIfExpression <- 'NULLIF' Parens(Expression ',' Expression)\n" - "PositionExpression <- 'POSITION' Parens(SingleExpression 'IN' SingleExpression)\n" + "NullIfExpression <- 'NULLIF' Parens(NullIfArguments)\n" + "NullIfArguments <- Expression ',' Expression\n" + "PositionExpression <- 'POSITION' Parens(PositionArguments)\n" + "PositionArguments <- SingleExpression 'IN' SingleExpression\n" "RowExpression <- 'ROW' Parens(List(Expression)?)\n" "SubstringExpression <- 'SUBSTRING' Parens(SubstringArguments)\n" "SubstringArguments <- SubstringParameters / SubstringExpressionList\n" @@ -954,7 +1011,8 @@ const char INLINED_PEG_GRAMMAR[] = { "SubstringFromFor <- SubstringFromOptionalFor / SubstringFor\n" "SubstringFromOptionalFor <- FromExpression ForExpression?\n" "SubstringFor <- ForExpression\n" - "TrimExpression <- 'TRIM' Parens(TrimDirection? TrimSource? List(Expression))\n" + "TrimExpression <- 'TRIM' Parens(TrimArguments)\n" + "TrimArguments <- TrimDirection? TrimSource? List(Expression)\n" "TrimDirection <- TrimBoth / TrimLeading / TrimTrailing\n" "TrimBoth <- 'BOTH'\n" "TrimLeading <- 'LEADING'\n" @@ -966,10 +1024,14 @@ const char INLINED_PEG_GRAMMAR[] = { "FromExpression <- 'FROM' Expression\n" "ForExpression <- 'FOR' Expression\n" "OverlayExpressionList <- List(Expression)\n" - "ExtractArgument <-\n" + "ExtractArgument <- ExtractDatePartArgument / ExtractIdentifierArgument / ExtractStringArgument\n" + "ExtractDatePartArgument <- ExtractDatePart\n" + "ExtractIdentifierArgument <- Identifier\n" + "ExtractStringArgument <- StringLiteral\n" + "ExtractDatePart <-\n" " YearKeyword / MonthKeyword / DayKeyword / HourKeyword / MinuteKeyword / SecondKeyword /\n" " MillisecondKeyword / MicrosecondKeyword / WeekKeyword / QuarterKeyword / DecadeKeyword /\n" - " CenturyKeyword / MillenniumKeyword / Identifier / StringLiteral\n" + " CenturyKeyword / MillenniumKeyword\n" "ExecuteStatement <- 'EXECUTE' Identifier TableFunctionArguments?\n" "CreateSecretStmt <- 'SECRET' IfNotExists? SecretName? SecretStorageSpecifier? GenericCopyOptionList\n" "SecretStorageSpecifier <- 'IN' Identifier\n" diff --git a/src/duckdb/src/include/duckdb/parser/peg/transformer/peg_transformer.hpp b/src/duckdb/src/include/duckdb/parser/peg/transformer/peg_transformer.hpp index b0539c264..a90fc6d71 100644 --- a/src/duckdb/src/include/duckdb/parser/peg/transformer/peg_transformer.hpp +++ b/src/duckdb/src/include/duckdb/parser/peg/transformer/peg_transformer.hpp @@ -28,6 +28,10 @@ #include "duckdb/parser/peg/ast/sequence_option.hpp" #include "duckdb/parser/peg/ast/setting_info.hpp" #include "duckdb/parser/peg/ast/table_alias.hpp" +#include "duckdb/parser/peg/ast/cast_arguments.hpp" +#include "duckdb/parser/peg/ast/expression_chain.hpp" +#include "duckdb/parser/peg/ast/method_arguments.hpp" +#include "duckdb/parser/peg/ast/trim_arguments.hpp" #include "duckdb/parser/peg/ast/trigger_event_info.hpp" #include "duckdb/parser/peg/ast/trigger_table_referencing_info.hpp" #include "duckdb/parser/peg/ast/window_frame.hpp" @@ -386,274 +390,11 @@ class PEGTransformerFactory { static string TransformColLabelOrString(PEGTransformer &transformer, ParseResult &parse_result); static string TransformIdentifier(PEGTransformer &transformer, ParseResult &parse_result); - // create_trigger.gram - static TriggerForEach TransformForEachClause(PEGTransformer &transformer, ParseResult &parse_result); - static TriggerTiming TransformTriggerTiming(PEGTransformer &transformer, ParseResult &parse_result); - static TriggerEventInfo TransformTriggerEvent(PEGTransformer &transformer, ParseResult &parse_result); - // expression.gram - static unique_ptr TransformExpressionStatement(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformExpressionAlias(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformBaseExpression(PEGTransformer &transformer, ParseResult &parse_result); static unique_ptr TransformExpression(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformLambdaArrowExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformLogicalOrExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformLogicalAndExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformLogicalNotExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformIsExpression(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformIsTest(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformIsLiteral(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformNotNull(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformIsNull(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformIsDistinctFromExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformComparisonExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static ExpressionType TransformComparisonOperator(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformBetweenInLikeExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformBetweenInLikeOp(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformInClause(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformInExpression(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformInExpressionList(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformInSelectStatement(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformBetweenClause(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformLikeClause(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformEscapeClause(PEGTransformer &transformer, ParseResult &parse_result); - static string TransformLikeVariations(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformOtherOperatorExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static string TransformOtherOperator(PEGTransformer &transformer, ParseResult &parse_result); - static string TransformQualifiedOperator(PEGTransformer &transformer, ParseResult &parse_result); - static string TransformAnyOp(PEGTransformer &transformer, ParseResult &parse_result); - static string TransformStringOperator(PEGTransformer &transformer, ParseResult &parse_result); - static string TransformJsonOperator(PEGTransformer &transformer, ParseResult &parse_result); - static string TransformInetOperator(PEGTransformer &transformer, ParseResult &parse_result); - static string TransformListOperator(PEGTransformer &transformer, ParseResult &parse_result); - static pair TransformAnyAllOperator(PEGTransformer &transformer, ParseResult &parse_result); - static bool TransformAnyOrAll(PEGTransformer &transformer, ParseResult &parse_result); - static ExpressionType TransformLambdaOperator(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformBitwiseExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static string TransformBitOperator(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformAdditiveExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static string TransformTerm(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformMultiplicativeExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static string TransformFactor(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformExponentiationExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static string TransformExponentOperator(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformPostfixOperator(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformCollateExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformAtTimeZoneExpression(PEGTransformer &transformer, - ParseResult &parse_result); static unique_ptr TransformPrefixExpression(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformConstantLiteral(PEGTransformer &transformer, - ParseResult &parse_result); - static Value TransformFalseLiteral(PEGTransformer &transformer, ParseResult &parse_result); - static Value TransformTrueLiteral(PEGTransformer &transformer, ParseResult &parse_result); - static Value TransformNullLiteral(PEGTransformer &transformer, ParseResult &parse_result); - static Value TransformUnknownLiteral(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformColumnReference(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformCatalogReservedSchemaTableColumnName(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformSchemaReservedTableColumnName(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformParameter(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformAnonymousParameter(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformQuestionMarkNumberedParameter(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformNumberedParameter(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformColLabelParameter(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformPositionalExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformLiteralExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformParensExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformSingleExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static string TransformPrefixOperator(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformListExpression(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformArrayBoundedListExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformArrayParensSelect(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformStructExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static FunctionArgument TransformStructField(PEGTransformer &transformer, ParseResult &parse_result); - static vector> TransformBoundedListExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformFunctionExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static QualifiedName TransformFunctionIdentifier(PEGTransformer &transformer, ParseResult &parse_result); - static QualifiedName TransformSchemaReservedFunctionName(PEGTransformer &transformer, ParseResult &parse_result); - static QualifiedName TransformCatalogReservedSchemaFunctionName(PEGTransformer &transformer, - ParseResult &parse_result); - static vector TransformWithinGroupClause(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformFilterClause(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformParenthesisExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformIndirection(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformCastOperator(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformDotOperator(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformMethodExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformSliceExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static vector> TransformSliceBound(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformEndSliceBound(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformStepSliceBound(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformTableReservedColumnName(PEGTransformer &transformer, - ParseResult &parse_result); - static string TransformColIdDot(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformStarExpression(PEGTransformer &transformer, ParseResult &parse_result); - static qualified_column_set_t TransformExcludeList(PEGTransformer &transformer, ParseResult &parse_result); - static qualified_column_set_t TransformExcludeNameList(PEGTransformer &transformer, ParseResult &parse_result); - static qualified_column_set_t TransformExcludeNameSingle(PEGTransformer &transformer, ParseResult &parse_result); - static QualifiedColumnName TransformExcludeName(PEGTransformer &transformer, ParseResult &parse_result); - static case_insensitive_map_t> TransformReplaceList(PEGTransformer &transformer, - ParseResult &parse_result); - static case_insensitive_map_t> TransformReplaceEntries(PEGTransformer &transformer, - ParseResult &parse_result); - static case_insensitive_map_t> TransformReplaceEntrySingle(PEGTransformer &transformer, - ParseResult &parse_result); - static case_insensitive_map_t> TransformReplaceEntryList(PEGTransformer &transformer, - ParseResult &parse_result); - static pair> TransformReplaceEntry(PEGTransformer &transformer, - ParseResult &parse_result); static unique_ptr TransformOverClause(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformWindowFrame(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformParensIdentifier(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformWindowFrameDefinition(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformWindowFrameContentsParens(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformWindowFrameNameContentsParens(PEGTransformer &transformer, - ParseResult &parse_result); - static string TransformBaseWindowName(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformWindowFrameContents(PEGTransformer &transformer, - ParseResult &parse_result); - - static WindowFrame TransformFrameClause(PEGTransformer &transformer, ParseResult &parse_result); - static string TransformFraming(PEGTransformer &transformer, ParseResult &parse_result); - static vector TransformFrameExtent(PEGTransformer &transformer, - ParseResult &parse_result); - static vector TransformBetweenFrameExtent(PEGTransformer &transformer, - ParseResult &parse_result); - static vector TransformSingleFrameExtent(PEGTransformer &transformer, - ParseResult &parse_result); - static WindowBoundaryExpression TransformFrameBound(PEGTransformer &transformer, ParseResult &parse_result); - static WindowBoundaryExpression TransformFrameUnbounded(PEGTransformer &transformer, ParseResult &parse_result); - static WindowBoundaryExpression TransformFrameCurrentRow(PEGTransformer &transformer, ParseResult &parse_result); - static WindowBoundaryExpression TransformFrameExpression(PEGTransformer &transformer, ParseResult &parse_result); - static bool TransformPrecedingOrFollowing(PEGTransformer &transformer, ParseResult &parse_result); - static WindowExcludeMode TransformWindowExcludeClause(PEGTransformer &transformer, ParseResult &parse_result); - static WindowExcludeMode TransformWindowExcludeElement(PEGTransformer &transformer, ParseResult &parse_result); - static vector> TransformWindowPartition(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformSpecialFunctionExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformCoalesceExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformUnpackExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformTryExpression(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformColumnsExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformExtractExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformExtractArgument(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformLambdaExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformNullIfExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformRowExpression(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformSubstringExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static vector> TransformSubstringParameters(PEGTransformer &transformer, - ParseResult &parse_result); - static vector> TransformSubstringFromFor(PEGTransformer &transformer, - ParseResult &parse_result); - static vector> TransformSubstringFromOptionalFor(PEGTransformer &transformer, - ParseResult &parse_result); - static vector> TransformSubstringFor(PEGTransformer &transformer, - ParseResult &parse_result); - static vector> TransformSubstringArguments(PEGTransformer &transformer, - ParseResult &parse_result); - static vector> TransformSubstringExpressionList(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformTrimExpression(PEGTransformer &transformer, ParseResult &parse_result); - static string TransformTrimDirection(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformTrimSource(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformOverlayExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static vector> TransformOverlayArguments(PEGTransformer &transformer, - ParseResult &parse_result); - static vector> TransformOverlayParameters(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformFromExpression(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformForExpression(PEGTransformer &transformer, ParseResult &parse_result); - static vector> TransformOverlayExpressionList(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformPositionExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformCastExpression(PEGTransformer &transformer, ParseResult &parse_result); - static bool TransformCastOrTryCast(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformCaseExpression(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformCaseElse(PEGTransformer &transformer, ParseResult &parse_result); - static CaseCheck TransformCaseWhenThen(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformTypeLiteral(PEGTransformer &transformer, ParseResult &parse_result); - static unique_ptr TransformDefaultExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformIntervalLiteral(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformIntervalParameter(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformSubqueryExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformMapExpression(PEGTransformer &transformer, ParseResult &parse_result); - static vector> TransformMapStructExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static vector> TransformMapStructField(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformListComprehensionExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static unique_ptr TransformListComprehensionFilter(PEGTransformer &transformer, - ParseResult &parse_result); - static ExpressionType TransformIsDistinctFromOp(PEGTransformer &transformer, ParseResult &parse_result); - - static unique_ptr TransformGroupingExpression(PEGTransformer &transformer, - ParseResult &parse_result); - static qualified_column_map_t TransformRenameList(PEGTransformer &transformer, ParseResult &parse_result); - static qualified_column_map_t TransformRenameEntryList(PEGTransformer &transformer, - ParseResult &parse_result); - static qualified_column_map_t TransformSingleRenameEntry(PEGTransformer &transformer, - ParseResult &parse_result); - static pair TransformRenameEntry(PEGTransformer &transformer, - ParseResult &parse_result); - static bool TransformIgnoreOrRespectNulls(PEGTransformer &transformer, ParseResult &parse_result); // pivot.gram static unique_ptr TransformPivotStatement(PEGTransformer &transformer, ParseResult &parse_result); @@ -663,9 +404,6 @@ class PEGTransformerFactory { // select.gram static unique_ptr TransformSelectStatementInternalRule(PEGTransformer &transformer, ParseResult &parse_result); - static bool TransformDistinctOrAll(PEGTransformer &transformer, ParseResult &parse_result); - static bool TransformDistinctKeyword(PEGTransformer &transformer, ParseResult &parse_result); - static bool TransformAllKeyword(PEGTransformer &transformer, ParseResult &parse_result); static unique_ptr TransformSimpleSelect(PEGTransformer &transformer, ParseResult &parse_result); static unique_ptr TransformTableRef(PEGTransformer &transformer, ParseResult &parse_result); @@ -2070,6 +1808,952 @@ class PEGTransformerFactory { static unique_ptr TransformImportStatementInternal(PEGTransformer &transformer, ParseResult &parse_result); static unique_ptr TransformImportStatement(PEGTransformer &transformer, const string &string_literal); + static unique_ptr TransformColumnReferenceInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformColumnReference(PEGTransformer &transformer, + unique_ptr child); + static unique_ptr + TransformCatalogReservedSchemaTableColumnNameInternal(PEGTransformer &transformer, ParseResult &parse_result); + static unique_ptr + TransformCatalogReservedSchemaTableColumnName(PEGTransformer &transformer, const Identifier &catalog_qualification, + const Identifier &reserved_schema_qualification, + const Identifier &reserved_table_qualification, + const Identifier &reserved_column_name); + static unique_ptr TransformSchemaReservedTableColumnNameInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformSchemaReservedTableColumnName(PEGTransformer &transformer, const Identifier &schema_qualification, + const Identifier &reserved_table_qualification, + const Identifier &reserved_column_name); + static unique_ptr TransformTableReservedColumnNameInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformTableReservedColumnName(PEGTransformer &transformer, + const Identifier &table_qualification, + const Identifier &reserved_column_name); + static unique_ptr TransformFunctionExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformFunctionExpression(PEGTransformer &transformer, const QualifiedName &function_identifier, + MethodArguments function_expression_arguments, + optional> within_group_clause, + optional> filter_clause, const bool &has_result, + optional> over_clause); + static unique_ptr TransformFunctionExpressionArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformFunctionExpressionArgumentListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static MethodArguments + TransformFunctionExpressionArgumentList(PEGTransformer &transformer, const optional &distinct_or_all, + optional> function_argument_list, + optional> order_by_clause, + const optional &ignore_or_respect_nulls); + static unique_ptr TransformFunctionArgumentListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformFunctionIdentifierInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static QualifiedName TransformFunctionIdentifier(PEGTransformer &transformer, ParseResult &choice_result); + static unique_ptr + TransformCatalogReservedSchemaFunctionNameInternal(PEGTransformer &transformer, ParseResult &parse_result); + static QualifiedName + TransformCatalogReservedSchemaFunctionName(PEGTransformer &transformer, const Identifier &catalog_qualification, + const optional &reserved_schema_qualification, + const Identifier &reserved_function_name); + static unique_ptr TransformSchemaReservedFunctionNameInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static QualifiedName TransformSchemaReservedFunctionName(PEGTransformer &transformer, + const Identifier &schema_qualification, + const Identifier &reserved_function_name); + static unique_ptr TransformDistinctOrAllInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformDistinctKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformDistinctKeyword(PEGTransformer &transformer); + static unique_ptr TransformAllKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformAllKeyword(PEGTransformer &transformer); + static unique_ptr TransformWithinGroupClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector TransformWithinGroupClause(PEGTransformer &transformer, + vector order_by_clause); + static unique_ptr TransformFilterClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformFilterClause(PEGTransformer &transformer, + unique_ptr filter_clause_expression); + static unique_ptr TransformFilterClauseExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformFilterClauseExpression(PEGTransformer &transformer, unique_ptr filter_clause_contents); + static unique_ptr TransformFilterClauseContentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformFilterClauseContents(PEGTransformer &transformer, + const bool &has_result, + unique_ptr expression); + static unique_ptr TransformIgnoreOrRespectNullsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformIgnoreNullsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformIgnoreNulls(PEGTransformer &transformer); + static unique_ptr TransformRespectNullsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformRespectNulls(PEGTransformer &transformer); + static unique_ptr TransformParenthesisExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformParenthesisExpression(PEGTransformer &transformer, + vector> expression); + static unique_ptr TransformLiteralExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformLiteralExpression(PEGTransformer &transformer, + ParseResult &choice_result); + static unique_ptr TransformConstantLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformConstantLiteral(PEGTransformer &transformer, const Value &child); + static unique_ptr TransformNullLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static Value TransformNullLiteral(PEGTransformer &transformer); + static unique_ptr TransformTrueLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static Value TransformTrueLiteral(PEGTransformer &transformer); + static unique_ptr TransformFalseLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static Value TransformFalseLiteral(PEGTransformer &transformer); + static unique_ptr TransformCastExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformCastExpression(PEGTransformer &transformer, const bool &cast_or_try_cast, CastArguments cast_arguments); + static unique_ptr TransformCastArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static CastArguments TransformCastArguments(PEGTransformer &transformer, unique_ptr expression, + const LogicalType &type); + static unique_ptr TransformCastOrTryCastInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformCastKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformCastKeyword(PEGTransformer &transformer); + static unique_ptr TransformTryCastKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformTryCastKeyword(PEGTransformer &transformer); + static unique_ptr TransformColIdDotInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformColIdDot(PEGTransformer &transformer, const Identifier &col_id); + static unique_ptr TransformStarExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformStarExpression(PEGTransformer &transformer, const optional> &star_qualifier_list, + const optional &exclude_list, + optional>> replace_list, + const optional> &rename_list); + static unique_ptr TransformStarQualifierListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformExcludeListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static qualified_column_set_t TransformExcludeList(PEGTransformer &transformer, + const qualified_column_set_t &exclude_names); + static unique_ptr TransformExcludeNamesInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformExcludeNameListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static qualified_column_set_t TransformExcludeNameList(PEGTransformer &transformer, + const vector &exclude_name); + static unique_ptr TransformExcludeNameSingleInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static qualified_column_set_t TransformExcludeNameSingle(PEGTransformer &transformer, + const QualifiedColumnName &exclude_name); + static unique_ptr TransformExcludeNameInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformExcludeDottedNameInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static QualifiedColumnName TransformExcludeDottedName(PEGTransformer &transformer, + const vector &dotted_identifier); + static unique_ptr TransformExcludeColumnNameInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static QualifiedColumnName TransformExcludeColumnName(PEGTransformer &transformer, + const Identifier &col_id_or_string); + static unique_ptr TransformReplaceListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static case_insensitive_map_t> + TransformReplaceList(PEGTransformer &transformer, + case_insensitive_map_t> replace_entries); + static unique_ptr TransformReplaceEntriesInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformReplaceEntrySingleInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static case_insensitive_map_t> + TransformReplaceEntrySingle(PEGTransformer &transformer, pair> replace_entry); + static unique_ptr TransformReplaceEntryListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static case_insensitive_map_t> + TransformReplaceEntryList(PEGTransformer &transformer, + vector>> replace_entry); + static unique_ptr TransformReplaceEntryInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static pair> + TransformReplaceEntry(PEGTransformer &transformer, unique_ptr expression, + unique_ptr column_reference); + static unique_ptr TransformRenameListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static qualified_column_map_t TransformRenameList(PEGTransformer &transformer, + const qualified_column_map_t &rename_entries); + static unique_ptr TransformRenameEntriesInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformRenameEntryListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static qualified_column_map_t + TransformRenameEntryList(PEGTransformer &transformer, + const vector> &rename_entry); + static unique_ptr TransformSingleRenameEntryInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static qualified_column_map_t + TransformSingleRenameEntry(PEGTransformer &transformer, const pair &rename_entry); + static unique_ptr TransformRenameEntryInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static pair TransformRenameEntry(PEGTransformer &transformer, + const QualifiedColumnName &exclude_name, + const Identifier &identifier); + static unique_ptr TransformSubqueryExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformSubqueryExpression(PEGTransformer &transformer, + const optional &subquery_not, + const optional &subquery_exists, + unique_ptr subquery_reference); + static unique_ptr TransformSubqueryNotInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformSubqueryNot(PEGTransformer &transformer); + static unique_ptr TransformSubqueryExistsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformSubqueryExists(PEGTransformer &transformer); + static unique_ptr TransformCaseExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformCaseExpression(PEGTransformer &transformer, + optional> expression, + vector case_when_then, + optional> case_else); + static unique_ptr TransformCaseWhenThenInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static CaseCheck TransformCaseWhenThen(PEGTransformer &transformer, unique_ptr expression, + unique_ptr expression_1); + static unique_ptr TransformCaseElseInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformCaseElse(PEGTransformer &transformer, + unique_ptr expression); + static unique_ptr TransformTypeLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformTypeLiteral(PEGTransformer &transformer, const Identifier &col_id, + const string &string_literal); + static unique_ptr TransformIntervalLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformIntervalLiteral(PEGTransformer &transformer, + unique_ptr interval_parameter, + const optional &interval); + static unique_ptr TransformIntervalParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformIntervalStringParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformIntervalStringParameter(PEGTransformer &transformer, + const string &string_literal); + static unique_ptr TransformFrameClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static WindowFrame TransformFrameClause(PEGTransformer &transformer, const string &framing, + vector frame_extent, + const optional &window_exclude_clause); + static unique_ptr TransformFramingInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformRowsFramingInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformRowsFraming(PEGTransformer &transformer); + static unique_ptr TransformRangeFramingInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformRangeFraming(PEGTransformer &transformer); + static unique_ptr TransformGroupsFramingInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformGroupsFraming(PEGTransformer &transformer); + static unique_ptr TransformFrameExtentInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformSingleFrameExtentInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector TransformSingleFrameExtent(PEGTransformer &transformer, + WindowBoundaryExpression frame_bound); + static unique_ptr TransformBetweenFrameExtentInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector TransformBetweenFrameExtent(PEGTransformer &transformer, + WindowBoundaryExpression frame_bound, + WindowBoundaryExpression frame_bound_1); + static unique_ptr TransformFrameBoundInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformFrameUnboundedInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static WindowBoundaryExpression TransformFrameUnbounded(PEGTransformer &transformer, + const bool &preceding_or_following); + static unique_ptr TransformFrameExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static WindowBoundaryExpression TransformFrameExpression(PEGTransformer &transformer, + unique_ptr expression, + const bool &preceding_or_following); + static unique_ptr TransformFrameCurrentRowInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static WindowBoundaryExpression TransformFrameCurrentRow(PEGTransformer &transformer); + static unique_ptr TransformPrecedingOrFollowingInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformPrecedingFrameInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformPrecedingFrame(PEGTransformer &transformer); + static unique_ptr TransformFollowingFrameInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformFollowingFrame(PEGTransformer &transformer); + static unique_ptr TransformWindowExcludeClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static WindowExcludeMode TransformWindowExcludeClause(PEGTransformer &transformer, + const WindowExcludeMode &window_exclude_element); + static unique_ptr TransformWindowExcludeElementInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformExcludeCurrentRowInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static WindowExcludeMode TransformExcludeCurrentRow(PEGTransformer &transformer); + static unique_ptr TransformExcludeGroupInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static WindowExcludeMode TransformExcludeGroup(PEGTransformer &transformer); + static unique_ptr TransformExcludeTiesInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static WindowExcludeMode TransformExcludeTies(PEGTransformer &transformer); + static unique_ptr TransformExcludeNoOthersInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static WindowExcludeMode TransformExcludeNoOthers(PEGTransformer &transformer); + static unique_ptr TransformWindowFrameInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformWindowFrame(PEGTransformer &transformer, ParseResult &choice_result); + static unique_ptr TransformParensIdentifierInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformParensIdentifier(PEGTransformer &transformer, + const Identifier &identifier); + static unique_ptr TransformWindowFrameDefinitionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformWindowFrameNameContentsParensInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformWindowFrameNameContentsParens(PEGTransformer &transformer, + unique_ptr window_frame_name_contents); + static unique_ptr TransformWindowFrameNameContentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformWindowFrameNameContents(PEGTransformer &transformer, const optional &base_window_name, + unique_ptr window_frame_contents); + static unique_ptr TransformWindowFrameContentsParensInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformWindowFrameContentsParens(PEGTransformer &transformer, unique_ptr window_frame_contents); + static unique_ptr TransformWindowFrameContentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformWindowFrameContents(PEGTransformer &transformer, + optional>> window_partition, + optional> order_by_clause, optional frame_clause); + static unique_ptr TransformBaseWindowNameInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static Identifier TransformBaseWindowName(PEGTransformer &transformer, const Identifier &identifier); + static unique_ptr TransformWindowPartitionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> + TransformWindowPartition(PEGTransformer &transformer, vector> expression); + static unique_ptr TransformListExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformArrayBoundedListExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformArrayBoundedListExpression(PEGTransformer &transformer, const bool &has_result, + vector> bounded_list_expression); + static unique_ptr TransformArrayParensSelectInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformArrayParensSelect(PEGTransformer &transformer, unique_ptr select_statement_internal); + static unique_ptr TransformBoundedListExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> + TransformBoundedListExpression(PEGTransformer &transformer, + optional>> expression); + static unique_ptr TransformStructExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformStructExpression(PEGTransformer &transformer, + vector struct_field); + static unique_ptr TransformStructFieldInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static FunctionArgument TransformStructField(PEGTransformer &transformer, const Identifier &col_id_or_string, + unique_ptr expression); + static unique_ptr TransformMapExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformMapExpression(PEGTransformer &transformer, vector> map_struct_expression); + static unique_ptr TransformMapStructExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> + TransformMapStructExpression(PEGTransformer &transformer, + optional>>> map_struct_field); + static unique_ptr TransformMapStructFieldInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> TransformMapStructField(PEGTransformer &transformer, + unique_ptr expression, + unique_ptr expression_1); + static unique_ptr TransformGroupingExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformGroupingExpression(PEGTransformer &transformer, const bool &grouping_or_grouping_id, + optional>> expression); + static unique_ptr TransformGroupingOrGroupingIdInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformGroupingKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformGroupingKeyword(PEGTransformer &transformer); + static unique_ptr TransformGroupingIdKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformGroupingIdKeyword(PEGTransformer &transformer); + static unique_ptr TransformParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformQuestionMarkNumberedParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformQuestionMarkNumberedParameter(PEGTransformer &transformer, unique_ptr number_literal); + static unique_ptr TransformAnonymousParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformAnonymousParameter(PEGTransformer &transformer); + static unique_ptr TransformNumberedParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformNumberedParameter(PEGTransformer &transformer, + unique_ptr number_literal); + static unique_ptr TransformColLabelParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformColLabelParameter(PEGTransformer &transformer, + const string &col_label); + static unique_ptr TransformPositionalExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformPositionalExpression(PEGTransformer &transformer, + unique_ptr number_literal); + static unique_ptr TransformDefaultExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformDefaultExpression(PEGTransformer &transformer); + static unique_ptr TransformListComprehensionExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformListComprehensionExpression(PEGTransformer &transformer, unique_ptr expression, + const vector &col_id_or_string, + unique_ptr expression_1, + optional> list_comprehension_filter); + static unique_ptr TransformListComprehensionFilterInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformListComprehensionFilter(PEGTransformer &transformer, + unique_ptr expression); + static unique_ptr TransformParensExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformParensExpression(PEGTransformer &transformer, + unique_ptr expression); + static unique_ptr TransformSingleExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformColumnDefaultExprInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformLambdaArrowExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformLambdaArrowExpression(PEGTransformer &transformer, unique_ptr logical_or_expression, + optional>> single_arrow_pair); + static unique_ptr TransformSingleArrowPairInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformLogicalOrExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformLogicalOrExpression(PEGTransformer &transformer, unique_ptr logical_and_expression, + optional>> logical_or_expression_tail); + static unique_ptr TransformLogicalOrExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformColDefOrExprInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformColDefOrExpr(PEGTransformer &transformer, unique_ptr col_def_and_expr, + optional>> col_def_or_expression_tail); + static unique_ptr TransformColDefOrExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformLogicalAndExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformLogicalAndExpression(PEGTransformer &transformer, unique_ptr logical_not_expression, + optional>> logical_and_expression_tail); + static unique_ptr TransformLogicalAndExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformColDefAndExprInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformColDefAndExpr(PEGTransformer &transformer, unique_ptr is_distinct_from_expression, + optional>> col_def_and_expression_tail); + static unique_ptr TransformColDefAndExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformLogicalNotExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformLogicalNotExpression(PEGTransformer &transformer, + optional> not_expression, + unique_ptr is_expression); + static unique_ptr TransformNotExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformNotKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformNotKeyword(PEGTransformer &transformer); + static unique_ptr TransformIsExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformIsExpression(PEGTransformer &transformer, + unique_ptr is_distinct_from_expression, + optional>> is_test); + static unique_ptr TransformIsTestInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformIsLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformIsLiteral(PEGTransformer &transformer, const bool &has_result, + const Value &is_literal_value); + static unique_ptr TransformIsLiteralValueInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformUnknownLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static Value TransformUnknownLiteral(PEGTransformer &transformer); + static unique_ptr TransformNotNullInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformNotNullKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformNotNullKeyword(PEGTransformer &transformer); + static unique_ptr TransformNotNullOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformNotNullOperator(PEGTransformer &transformer); + static unique_ptr TransformIsNullInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformIsNullOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformIsNullOperator(PEGTransformer &transformer); + static unique_ptr TransformIsDistinctFromExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformIsDistinctFromExpression(PEGTransformer &transformer, unique_ptr comparison_expression, + optional> is_distinct_from_tail); + static unique_ptr TransformIsDistinctFromTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static IsDistinctFromTail TransformIsDistinctFromTail(PEGTransformer &transformer, + const ExpressionType &is_distinct_from_op, + unique_ptr comparison_expression); + static unique_ptr TransformIsDistinctFromOpInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static ExpressionType TransformIsDistinctFromOp(PEGTransformer &transformer, const bool &has_result); + static unique_ptr TransformComparisonExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformComparisonExpression(PEGTransformer &transformer, unique_ptr between_in_like_expression, + optional> comparison_expression_tail); + static unique_ptr TransformComparisonExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static ComparisonExpressionTail + TransformComparisonExpressionTail(PEGTransformer &transformer, const ExpressionType &comparison_operator, + optional> not_expression, + unique_ptr between_in_like_expression); + static unique_ptr TransformComparisonOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformOperatorEqualInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static ExpressionType TransformOperatorEqual(PEGTransformer &transformer); + static unique_ptr TransformOperatorNotEqualInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static ExpressionType TransformOperatorNotEqual(PEGTransformer &transformer); + static unique_ptr TransformOperatorLessThanInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static ExpressionType TransformOperatorLessThan(PEGTransformer &transformer); + static unique_ptr TransformOperatorGreaterThanInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static ExpressionType TransformOperatorGreaterThan(PEGTransformer &transformer); + static unique_ptr TransformOperatorLessThanEqualsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static ExpressionType TransformOperatorLessThanEquals(PEGTransformer &transformer); + static unique_ptr TransformOperatorGreaterThanEqualsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static ExpressionType TransformOperatorGreaterThanEquals(PEGTransformer &transformer); + static unique_ptr TransformBetweenInLikeExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformBetweenInLikeExpression(PEGTransformer &transformer, + unique_ptr other_operator_expression, + optional between_in_like_op); + static unique_ptr TransformBetweenInLikeOpInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static BetweenInLikeOperator TransformBetweenInLikeOp(PEGTransformer &transformer, const bool &has_result, + unique_ptr between_in_like_op_expression); + static unique_ptr TransformBetweenInLikeOpExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformLikeClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformLikeClause(PEGTransformer &transformer, const string &like_variations, + unique_ptr other_operator_expression, + optional> escape_clause); + static unique_ptr TransformEscapeClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformEscapeClause(PEGTransformer &transformer, + unique_ptr comparison_expression); + static unique_ptr TransformLikeVariationsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformLikeTokenInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformLikeToken(PEGTransformer &transformer); + static unique_ptr TransformILikeTokenInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformILikeToken(PEGTransformer &transformer); + static unique_ptr TransformGlobTokenInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformGlobToken(PEGTransformer &transformer); + static unique_ptr TransformSimilarToTokenInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformSimilarToToken(PEGTransformer &transformer); + static unique_ptr TransformNotILikeOpInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformNotILikeOp(PEGTransformer &transformer); + static unique_ptr TransformNotLikeOpInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformNotLikeOp(PEGTransformer &transformer); + static unique_ptr TransformNotSimilarToOpInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformNotSimilarToOp(PEGTransformer &transformer); + static unique_ptr TransformInClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformInClause(PEGTransformer &transformer, + unique_ptr in_expression); + static unique_ptr TransformInExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformInContainsExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformInContainsExpression(PEGTransformer &transformer, unique_ptr other_operator_expression); + static unique_ptr TransformInExpressionListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformInExpressionList(PEGTransformer &transformer, + vector> expression); + static unique_ptr TransformInSelectStatementInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformInSelectStatement(PEGTransformer &transformer, unique_ptr select_statement_internal); + static unique_ptr TransformBetweenClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformBetweenClause(PEGTransformer &transformer, unique_ptr other_operator_expression, + unique_ptr other_operator_expression_1); + static unique_ptr TransformOtherOperatorExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformOtherOperatorExpression(PEGTransformer &transformer, unique_ptr bitwise_expression, + optional> other_operator_tail); + static unique_ptr TransformOtherOperatorTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static OtherOperatorTail TransformOtherOperatorTail(PEGTransformer &transformer, ParsedOperator other_operator, + unique_ptr bitwise_expression); + static unique_ptr TransformOtherOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static ParsedOperator TransformOtherOperator(PEGTransformer &transformer, ParseResult &choice_result); + static unique_ptr TransformAnyAllOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static pair TransformAnyAllOperator(PEGTransformer &transformer, const string &any_op, + const bool &any_or_all); + static unique_ptr TransformAnyOrAllInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformSubqueryAnyInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformSubqueryAny(PEGTransformer &transformer); + static unique_ptr TransformSubqueryAllInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static bool TransformSubqueryAll(PEGTransformer &transformer); + static unique_ptr TransformInetOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformJsonOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformListOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformStringOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformQualifiedOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformQualifiedOperator(PEGTransformer &transformer, const string &qualified_operator_contents); + static unique_ptr TransformQualifiedOperatorContentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformQualifiedOperatorContents(PEGTransformer &transformer, + const optional> &col_id_dot, const string &any_op); + static unique_ptr TransformAnyOpInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformBitwiseExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformBitwiseExpression(PEGTransformer &transformer, unique_ptr additive_expression, + optional> bitwise_expression_tail); + static unique_ptr TransformBitwiseExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static BinaryExpressionTail TransformBitwiseExpressionTail(PEGTransformer &transformer, const string &bit_operator, + unique_ptr additive_expression); + static unique_ptr TransformBitOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformAdditiveExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformAdditiveExpression(PEGTransformer &transformer, unique_ptr multiplicative_expression, + optional> additive_expression_tail); + static unique_ptr TransformAdditiveExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static BinaryExpressionTail TransformAdditiveExpressionTail(PEGTransformer &transformer, const string &term, + unique_ptr multiplicative_expression, + optional_idx query_location); + static unique_ptr TransformTermInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformMultiplicativeExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformMultiplicativeExpression(PEGTransformer &transformer, + unique_ptr exponentiation_expression, + optional> multiplicative_expression_tail); + static unique_ptr TransformMultiplicativeExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static BinaryExpressionTail + TransformMultiplicativeExpressionTail(PEGTransformer &transformer, const string &factor, + unique_ptr exponentiation_expression); + static unique_ptr TransformFactorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformExponentiationExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformExponentiationExpression(PEGTransformer &transformer, unique_ptr collate_expression, + optional> exponentiation_expression_tail); + static unique_ptr TransformExponentiationExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static BinaryExpressionTail TransformExponentiationExpressionTail(PEGTransformer &transformer, + const string &exponent_operator, + unique_ptr collate_expression); + static unique_ptr TransformExponentOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformCollateExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformCollateExpression(PEGTransformer &transformer, unique_ptr at_time_zone_expression, + optional>> collate_expression_tail); + static unique_ptr TransformCollateExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformAtTimeZoneExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformAtTimeZoneExpression(PEGTransformer &transformer, unique_ptr prefix_expression, + optional>> at_time_zone_expression_tail); + static unique_ptr TransformAtTimeZoneExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformPrefixOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformMinusPrefixOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformPlusPrefixOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformTildePrefixOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformBaseExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformBaseExpression(PEGTransformer &transformer, unique_ptr single_expression, + optional>> indirection_list); + static unique_ptr TransformIndirectionListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformIndirectionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformCastOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformCastOperator(PEGTransformer &transformer, const LogicalType &type); + static unique_ptr TransformDotOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformDotMethodOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformDotMethodOperator(PEGTransformer &transformer, + unique_ptr method_expression); + static unique_ptr TransformDotColumnOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformDotColumnOperator(PEGTransformer &transformer, + const string &col_label); + static unique_ptr TransformMethodExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformMethodExpression(PEGTransformer &transformer, const string &col_label, + MethodArguments method_expression_arguments); + static unique_ptr TransformMethodExpressionArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static MethodArguments TransformMethodExpressionArguments(PEGTransformer &transformer, + MethodArguments method_expression_argument_list); + static unique_ptr TransformMethodExpressionArgumentListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static MethodArguments + TransformMethodExpressionArgumentList(PEGTransformer &transformer, const optional &distinct_or_all, + optional> method_function_arguments, + optional> order_by_clause, + const optional &ignore_or_respect_nulls); + static unique_ptr TransformMethodFunctionArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector TransformMethodFunctionArguments(PEGTransformer &transformer, + vector function_argument); + static unique_ptr TransformSliceExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformSliceExpression(PEGTransformer &transformer, + vector> slice_bound); + static unique_ptr TransformSliceBoundInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> + TransformSliceBound(PEGTransformer &transformer, optional> expression, + optional> end_slice_bound, + optional> step_slice_bound); + static unique_ptr TransformEndSliceBoundInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformEndSliceBound(PEGTransformer &transformer, + optional> end_slice_value); + static unique_ptr TransformEndSliceValueInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformEndSliceMinusInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformEndSliceMinus(PEGTransformer &transformer); + static unique_ptr TransformStepSliceBoundInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformStepSliceBound(PEGTransformer &transformer, + optional> expression); + static unique_ptr TransformPostfixOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformPostfixOperator(PEGTransformer &transformer); + static unique_ptr TransformSpecialFunctionExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformCoalesceExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformCoalesceExpression(PEGTransformer &transformer, + vector> expression); + static unique_ptr TransformUnpackExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformUnpackExpression(PEGTransformer &transformer, + unique_ptr expression); + static unique_ptr TransformTryExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformTryExpression(PEGTransformer &transformer, + unique_ptr expression); + static unique_ptr TransformColumnsExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformColumnsExpression(PEGTransformer &transformer, const bool &has_result, + unique_ptr expression); + static unique_ptr TransformExtractExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformExtractExpression(PEGTransformer &transformer, vector> extract_arguments); + static unique_ptr TransformExtractArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> TransformExtractArguments(PEGTransformer &transformer, + unique_ptr extract_argument, + unique_ptr expression); + static unique_ptr TransformLambdaExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformLambdaExpression(PEGTransformer &transformer, + const vector &col_id_or_string, + unique_ptr expression); + static unique_ptr TransformNullIfExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformNullIfExpression(PEGTransformer &transformer, vector> null_if_arguments); + static unique_ptr TransformNullIfArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> TransformNullIfArguments(PEGTransformer &transformer, + unique_ptr expression, + unique_ptr expression_1); + static unique_ptr TransformPositionExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformPositionExpression(PEGTransformer &transformer, vector> position_arguments); + static unique_ptr TransformPositionArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> + TransformPositionArguments(PEGTransformer &transformer, unique_ptr single_expression, + unique_ptr single_expression_1); + static unique_ptr TransformRowExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformRowExpression(PEGTransformer &transformer, optional>> expression); + static unique_ptr TransformSubstringExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformSubstringExpression(PEGTransformer &transformer, vector> substring_arguments); + static unique_ptr TransformSubstringArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformSubstringExpressionListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> + TransformSubstringExpressionList(PEGTransformer &transformer, vector> expression); + static unique_ptr TransformSubstringParametersInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> + TransformSubstringParameters(PEGTransformer &transformer, unique_ptr expression, + vector> substring_from_for); + static unique_ptr TransformSubstringFromForInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformSubstringFromOptionalForInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> + TransformSubstringFromOptionalFor(PEGTransformer &transformer, unique_ptr from_expression, + optional> for_expression); + static unique_ptr TransformSubstringForInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> TransformSubstringFor(PEGTransformer &transformer, + unique_ptr for_expression); + static unique_ptr TransformTrimExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformTrimExpression(PEGTransformer &transformer, + TrimArguments trim_arguments); + static unique_ptr TransformTrimArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static TrimArguments TransformTrimArguments(PEGTransformer &transformer, const optional &trim_direction, + optional> trim_source, + vector> expression); + static unique_ptr TransformTrimDirectionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformTrimBothInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformTrimBoth(PEGTransformer &transformer); + static unique_ptr TransformTrimLeadingInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformTrimLeading(PEGTransformer &transformer); + static unique_ptr TransformTrimTrailingInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static string TransformTrimTrailing(PEGTransformer &transformer); + static unique_ptr TransformTrimSourceInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformTrimSource(PEGTransformer &transformer, + optional> expression); + static unique_ptr TransformOverlayExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr + TransformOverlayExpression(PEGTransformer &transformer, vector> overlay_arguments); + static unique_ptr TransformOverlayArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformOverlayParametersInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> + TransformOverlayParameters(PEGTransformer &transformer, unique_ptr expression, + unique_ptr expression_1, unique_ptr from_expression, + optional> for_expression); + static unique_ptr TransformFromExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformFromExpression(PEGTransformer &transformer, + unique_ptr expression); + static unique_ptr TransformForExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformForExpression(PEGTransformer &transformer, + unique_ptr expression); + static unique_ptr TransformOverlayExpressionListInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static vector> + TransformOverlayExpressionList(PEGTransformer &transformer, vector> expression); + static unique_ptr TransformExtractArgumentInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformExtractDatePartArgumentInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformExtractDatePartArgument(PEGTransformer &transformer, + const DatePartSpecifier &extract_date_part); + static unique_ptr TransformExtractIdentifierArgumentInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformExtractIdentifierArgument(PEGTransformer &transformer, + const Identifier &identifier); + static unique_ptr TransformExtractStringArgumentInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static unique_ptr TransformExtractStringArgument(PEGTransformer &transformer, + const string &string_literal); + static unique_ptr TransformExtractDatePartInternal(PEGTransformer &transformer, + ParseResult &parse_result); static unique_ptr TransformInsertStatementInternal(PEGTransformer &transformer, ParseResult &parse_result); static unique_ptr diff --git a/src/duckdb/src/include/duckdb/storage/external_file_cache/caching_file_system.hpp b/src/duckdb/src/include/duckdb/storage/external_file_cache/caching_file_system.hpp index 9deafa37c..b2024856e 100644 --- a/src/duckdb/src/include/duckdb/storage/external_file_cache/caching_file_system.hpp +++ b/src/duckdb/src/include/duckdb/storage/external_file_cache/caching_file_system.hpp @@ -32,6 +32,23 @@ struct NetworkThroughputEstimate; class QueryContext; class CachingFileSystem; +struct ReadThroughputEstimator { + //! Record a timed read. + void AddSample(double seconds, idx_t bytes); + //! Try to produce an estimate. Needs reads of at least two different sizes to separate latency from bandwidth. + bool TryEstimate(NetworkThroughputEstimate &result) const; + +private: + mutable mutex lock; + idx_t sample_count = 0; + double sum_bytes = 0; + double sum_seconds = 0; + //! sum of bytes squared + double sum_bytes_sq = 0; + //! sum of bytes times seconds + double sum_bytes_seconds = 0; +}; + struct CachingFileHandle { public: using CachedFile = ExternalFileCache::CachedFile; @@ -51,6 +68,8 @@ struct CachingFileHandle { DUCKDB_API FileBufferHandleGroup Read(idx_t nr_bytes, idx_t location); //! Read [nr_bytes] bytes and sets [nr_bytes] to the actually read bytes. DUCKDB_API FileBufferHandleGroup Read(idx_t &nr_bytes); + //! Read and record time + DUCKDB_API void ReadAndRecord(QueryContext context, data_ptr_t buffer, idx_t nr_bytes, idx_t location); //! Get some properties of the file DUCKDB_API string GetPath() const; DUCKDB_API idx_t GetFileSize(); @@ -69,6 +88,8 @@ struct CachingFileHandle { bool StripForceFullDownloadIfPresent(); //! Refresh the cached file if the global cache state has changed. shared_ptr EnsureCachedFileCurrent(); + //! Record a timed read of a local file into the throughput estimate + void RecordReadThroughput(double total_seconds, idx_t bytes); private: QueryContext context; @@ -98,6 +119,9 @@ struct CachingFileHandle { //! Current position (if non-seeking reads) idx_t position; + + //! Throughput fitted from this handle's own reads, used for local files (remote files measure their own). + ReadThroughputEstimator throughput_estimator; }; //! CachingFileSystem is a read-only file system that closely resembles the FileSystem API. diff --git a/src/duckdb/src/include/duckdb/storage/external_file_cache/external_file_cache.hpp b/src/duckdb/src/include/duckdb/storage/external_file_cache/external_file_cache.hpp index ac562780d..65725aa36 100644 --- a/src/duckdb/src/include/duckdb/storage/external_file_cache/external_file_cache.hpp +++ b/src/duckdb/src/include/duckdb/storage/external_file_cache/external_file_cache.hpp @@ -35,6 +35,8 @@ class ExternalFileCache { public: //! Get the cache block size for a given file path. DUCKDB_API idx_t GetCacheBlockSize(const string &path) const; + //! Whether reads of the given file should go through the cache (remote files only, unless forced). + DUCKDB_API bool ShouldCacheFile(const string &path) const; //! Cached files struct CachedFile { diff --git a/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp b/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp index 603964509..8d0ded7fe 100644 --- a/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp @@ -71,8 +71,10 @@ class ColumnSegment : public SegmentBase { //! Fetch a value of the specific row id and append it to the result void FetchRow(ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx); - static idx_t FilterSelection(SelectionVector &sel, Vector &vector, UnifiedVectorFormat &vdata, - const TableFilter &filter, TableFilterState &filter_state, idx_t scan_count, + [[deprecated("UnifiedVectorFormat parameter is ignored")]] static idx_t + FilterSelection(SelectionVector &sel, Vector &vector, UnifiedVectorFormat &vdata, const TableFilter &filter, + TableFilterState &filter_state, idx_t scan_count, idx_t &approved_tuple_count); + static idx_t FilterSelection(SelectionVector &sel, Vector &vector, TableFilterState &filter_state, idx_t scan_count, idx_t &approved_tuple_count); //! Skip a scan forward to the row_index specified in the scan state diff --git a/src/duckdb/src/main/config.cpp b/src/duckdb/src/main/config.cpp index 86d5b59da..0d24bfc6c 100644 --- a/src/duckdb/src/main/config.cpp +++ b/src/duckdb/src/main/config.cpp @@ -91,6 +91,7 @@ static const ConfigurationOption internal_options[] = { DUCKDB_SETTING(AutoinstallKnownExtensionsSetting), DUCKDB_SETTING(AutoloadKnownExtensionsSetting), DUCKDB_GLOBAL(BlockAllocatorMemorySetting), + DUCKDB_SETTING(CacheLocalFilesSetting), DUCKDB_SETTING(CatalogErrorMaxSchemasSetting), DUCKDB_SETTING_CALLBACK(CheckpointOnDetachSetting), DUCKDB_GLOBAL(CheckpointThresholdSetting), @@ -106,6 +107,7 @@ static const ConfigurationOption internal_options[] = { DUCKDB_SETTING(DebugForceExternalSetting), DUCKDB_SETTING(DebugForceFetchRowSetting), DUCKDB_SETTING(DebugForceNoCrossProductSetting), + DUCKDB_GLOBAL(DebugOrderVerificationSetting), DUCKDB_SETTING_CALLBACK(DebugPhysicalTableScanExecutionStrategySetting), DUCKDB_SETTING(DebugSkipCheckpointOnCommitSetting), DUCKDB_GLOBAL(DebugVerificationModeSetting), @@ -238,14 +240,14 @@ static const ConfigurationOption internal_options[] = { DUCKDB_SETTING(ZstdMinStringLengthSetting), FINAL_SETTING}; -static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("configure_metrics", 28), - DUCKDB_SETTING_ALIAS("custom_profiling_settings", 28), - DUCKDB_SETTING_ALIAS("memory_limit", 122), - DUCKDB_SETTING_ALIAS("null_order", 55), - DUCKDB_SETTING_ALIAS("profile_output", 145), - DUCKDB_SETTING_ALIAS("user", 161), - DUCKDB_SETTING_ALIAS("wal_autocheckpoint", 27), - DUCKDB_SETTING_ALIAS("worker_threads", 159), +static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("configure_metrics", 29), + DUCKDB_SETTING_ALIAS("custom_profiling_settings", 29), + DUCKDB_SETTING_ALIAS("memory_limit", 124), + DUCKDB_SETTING_ALIAS("null_order", 57), + DUCKDB_SETTING_ALIAS("profile_output", 147), + DUCKDB_SETTING_ALIAS("user", 163), + DUCKDB_SETTING_ALIAS("wal_autocheckpoint", 28), + DUCKDB_SETTING_ALIAS("worker_threads", 161), FINAL_ALIAS}; vector DBConfig::GetOptions() { diff --git a/src/duckdb/src/main/settings/autogenerated_settings.cpp b/src/duckdb/src/main/settings/autogenerated_settings.cpp index ddf55ee9d..4b9b8c79d 100644 --- a/src/duckdb/src/main/settings/autogenerated_settings.cpp +++ b/src/duckdb/src/main/settings/autogenerated_settings.cpp @@ -77,6 +77,23 @@ void DebugCheckpointAbortSetting::OnSet(SettingCallbackInfo &info, Value ¶me EnumUtil::FromString(StringValue::Get(parameter)); } +//===----------------------------------------------------------------------===// +// Debug Order Verification +//===----------------------------------------------------------------------===// +void DebugOrderVerificationSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { + auto str_input = StringUtil::Upper(input.GetValue()); + config.options.debug_order_verification = EnumUtil::FromString(str_input); +} + +void DebugOrderVerificationSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { + config.options.debug_order_verification = DBConfigOptions().debug_order_verification; +} + +Value DebugOrderVerificationSetting::GetSetting(const ClientContext &context) { + auto &config = DBConfig::GetConfig(context); + return Value(StringUtil::Lower(EnumUtil::ToString(config.options.debug_order_verification))); +} + //===----------------------------------------------------------------------===// // Debug Physical Table Scan Execution Strategy //===----------------------------------------------------------------------===// diff --git a/src/duckdb/src/parser/peg/transformer/peg_transformer_factory.cpp b/src/duckdb/src/parser/peg/transformer/peg_transformer_factory.cpp index e19b85832..f4a5091c7 100644 --- a/src/duckdb/src/parser/peg/transformer/peg_transformer_factory.cpp +++ b/src/duckdb/src/parser/peg/transformer/peg_transformer_factory.cpp @@ -141,187 +141,9 @@ void PEGTransformerFactory::RegisterCreateTable() { void PEGTransformerFactory::RegisterExpression() { // expression.gram - REGISTER_TRANSFORM(TransformBaseExpression); REGISTER_TRANSFORM(TransformExpression); - Register("ColumnDefaultExpr", &TransformExpression); - Register("ColDefOrExpr", &TransformLogicalOrExpression); - Register("ColDefAndExpr", &TransformLogicalAndExpression); - REGISTER_TRANSFORM(TransformLambdaArrowExpression); - REGISTER_TRANSFORM(TransformLogicalOrExpression); - REGISTER_TRANSFORM(TransformLogicalAndExpression); - REGISTER_TRANSFORM(TransformLogicalNotExpression); - REGISTER_TRANSFORM(TransformIsExpression); - REGISTER_TRANSFORM(TransformIsTest); - REGISTER_TRANSFORM(TransformIsLiteral); - REGISTER_TRANSFORM(TransformNotNull); - REGISTER_TRANSFORM(TransformIsNull); - REGISTER_TRANSFORM(TransformIsDistinctFromExpression); - REGISTER_TRANSFORM(TransformBetweenInLikeExpression); - REGISTER_TRANSFORM(TransformBetweenInLikeOp); - REGISTER_TRANSFORM(TransformInClause); - REGISTER_TRANSFORM(TransformInExpression); - REGISTER_TRANSFORM(TransformInExpressionList); - REGISTER_TRANSFORM(TransformInSelectStatement); - REGISTER_TRANSFORM(TransformBetweenClause); - REGISTER_TRANSFORM(TransformLikeClause); - REGISTER_TRANSFORM(TransformEscapeClause); - REGISTER_TRANSFORM(TransformLikeVariations); - REGISTER_TRANSFORM(TransformComparisonExpression); - REGISTER_TRANSFORM(TransformComparisonOperator); - REGISTER_TRANSFORM(TransformOtherOperatorExpression); - REGISTER_TRANSFORM(TransformOtherOperator); - REGISTER_TRANSFORM(TransformQualifiedOperator); - REGISTER_TRANSFORM(TransformAnyOp); - REGISTER_TRANSFORM(TransformStringOperator); - REGISTER_TRANSFORM(TransformJsonOperator); - REGISTER_TRANSFORM(TransformInetOperator); - REGISTER_TRANSFORM(TransformAnyAllOperator); - REGISTER_TRANSFORM(TransformAnyOrAll); - REGISTER_TRANSFORM(TransformListOperator); - REGISTER_TRANSFORM(TransformLambdaOperator); - REGISTER_TRANSFORM(TransformBitwiseExpression); - REGISTER_TRANSFORM(TransformBitOperator); - REGISTER_TRANSFORM(TransformAdditiveExpression); - REGISTER_TRANSFORM(TransformTerm); - REGISTER_TRANSFORM(TransformMultiplicativeExpression); - REGISTER_TRANSFORM(TransformFactor); - REGISTER_TRANSFORM(TransformExponentiationExpression); - REGISTER_TRANSFORM(TransformExponentOperator); - REGISTER_TRANSFORM(TransformPostfixOperator); - REGISTER_TRANSFORM(TransformCollateExpression); - REGISTER_TRANSFORM(TransformAtTimeZoneExpression); REGISTER_TRANSFORM(TransformPrefixExpression); - - REGISTER_TRANSFORM(TransformColumnReference); - REGISTER_TRANSFORM(TransformCatalogReservedSchemaTableColumnName); - REGISTER_TRANSFORM(TransformSchemaReservedTableColumnName); - - REGISTER_TRANSFORM(TransformParameter); - REGISTER_TRANSFORM(TransformAnonymousParameter); - REGISTER_TRANSFORM(TransformQuestionMarkNumberedParameter); - REGISTER_TRANSFORM(TransformColLabelParameter); - REGISTER_TRANSFORM(TransformNumberedParameter); - REGISTER_TRANSFORM(TransformPositionalExpression); - - REGISTER_TRANSFORM(TransformLiteralExpression); - REGISTER_TRANSFORM(TransformParensExpression); - REGISTER_TRANSFORM(TransformSingleExpression); - REGISTER_TRANSFORM(TransformConstantLiteral); - REGISTER_TRANSFORM(TransformFalseLiteral); - REGISTER_TRANSFORM(TransformTrueLiteral); - REGISTER_TRANSFORM(TransformNullLiteral); - REGISTER_TRANSFORM(TransformUnknownLiteral); - - REGISTER_TRANSFORM(TransformPrefixOperator); - REGISTER_TRANSFORM(TransformListExpression); - REGISTER_TRANSFORM(TransformStructExpression); - REGISTER_TRANSFORM(TransformStructField); - REGISTER_TRANSFORM(TransformBoundedListExpression); - REGISTER_TRANSFORM(TransformArrayBoundedListExpression); - REGISTER_TRANSFORM(TransformArrayParensSelect); - REGISTER_TRANSFORM(TransformFunctionExpression); - REGISTER_TRANSFORM(TransformWithinGroupClause); - REGISTER_TRANSFORM(TransformFilterClause); - REGISTER_TRANSFORM(TransformFunctionIdentifier); - REGISTER_TRANSFORM(TransformSchemaReservedFunctionName); - REGISTER_TRANSFORM(TransformCatalogReservedSchemaFunctionName); - REGISTER_TRANSFORM(TransformParenthesisExpression); - REGISTER_TRANSFORM(TransformIndirection); - REGISTER_TRANSFORM(TransformCastOperator); - REGISTER_TRANSFORM(TransformDotOperator); - REGISTER_TRANSFORM(TransformSliceExpression); - REGISTER_TRANSFORM(TransformSliceBound); - REGISTER_TRANSFORM(TransformEndSliceBound); - REGISTER_TRANSFORM(TransformStepSliceBound); - - REGISTER_TRANSFORM(TransformTableReservedColumnName); - REGISTER_TRANSFORM(TransformColIdDot); - REGISTER_TRANSFORM(TransformStarExpression); - REGISTER_TRANSFORM(TransformExcludeList); - REGISTER_TRANSFORM(TransformExcludeNameList); - REGISTER_TRANSFORM(TransformExcludeNameSingle); - REGISTER_TRANSFORM(TransformExcludeName); - REGISTER_TRANSFORM(TransformReplaceList); - REGISTER_TRANSFORM(TransformReplaceEntries); - REGISTER_TRANSFORM(TransformReplaceEntrySingle); - REGISTER_TRANSFORM(TransformReplaceEntryList); - REGISTER_TRANSFORM(TransformReplaceEntry); - REGISTER_TRANSFORM(TransformOverClause); - REGISTER_TRANSFORM(TransformWindowFrame); - REGISTER_TRANSFORM(TransformParensIdentifier); - REGISTER_TRANSFORM(TransformWindowFrameDefinition); - REGISTER_TRANSFORM(TransformWindowFrameContentsParens); - REGISTER_TRANSFORM(TransformWindowFrameNameContentsParens); - REGISTER_TRANSFORM(TransformBaseWindowName); - REGISTER_TRANSFORM(TransformWindowFrameContents); - REGISTER_TRANSFORM(TransformFraming); - REGISTER_TRANSFORM(TransformFrameExtent); - REGISTER_TRANSFORM(TransformBetweenFrameExtent); - REGISTER_TRANSFORM(TransformSingleFrameExtent); - REGISTER_TRANSFORM(TransformFrameBound); - REGISTER_TRANSFORM(TransformFrameUnbounded); - REGISTER_TRANSFORM(TransformFrameCurrentRow); - REGISTER_TRANSFORM(TransformFrameExpression); - REGISTER_TRANSFORM(TransformPrecedingOrFollowing); - - REGISTER_TRANSFORM(TransformFrameClause); - REGISTER_TRANSFORM(TransformWindowExcludeClause); - REGISTER_TRANSFORM(TransformWindowExcludeElement); - - REGISTER_TRANSFORM(TransformWindowPartition); - - REGISTER_TRANSFORM(TransformSpecialFunctionExpression); - REGISTER_TRANSFORM(TransformCoalesceExpression); - REGISTER_TRANSFORM(TransformUnpackExpression); - REGISTER_TRANSFORM(TransformTryExpression); - REGISTER_TRANSFORM(TransformColumnsExpression); - REGISTER_TRANSFORM(TransformExtractExpression); - REGISTER_TRANSFORM(TransformExtractArgument); - REGISTER_TRANSFORM(TransformLambdaExpression); - REGISTER_TRANSFORM(TransformNullIfExpression); - REGISTER_TRANSFORM(TransformRowExpression); - REGISTER_TRANSFORM(TransformSubstringExpression); - REGISTER_TRANSFORM(TransformSubstringArguments); - REGISTER_TRANSFORM(TransformSubstringExpressionList); - REGISTER_TRANSFORM(TransformSubstringParameters); - REGISTER_TRANSFORM(TransformSubstringFromFor); - REGISTER_TRANSFORM(TransformSubstringFromOptionalFor); - REGISTER_TRANSFORM(TransformSubstringFor); - REGISTER_TRANSFORM(TransformTrimExpression); - REGISTER_TRANSFORM(TransformTrimDirection); - REGISTER_TRANSFORM(TransformTrimSource); - REGISTER_TRANSFORM(TransformOverlayExpression); - REGISTER_TRANSFORM(TransformOverlayArguments); - REGISTER_TRANSFORM(TransformOverlayParameters); - REGISTER_TRANSFORM(TransformFromExpression); - REGISTER_TRANSFORM(TransformForExpression); - REGISTER_TRANSFORM(TransformOverlayExpressionList); - REGISTER_TRANSFORM(TransformPositionExpression); - REGISTER_TRANSFORM(TransformCastExpression); - REGISTER_TRANSFORM(TransformCastOrTryCast); - REGISTER_TRANSFORM(TransformCaseExpression); - REGISTER_TRANSFORM(TransformCaseElse); - REGISTER_TRANSFORM(TransformCaseWhenThen); - REGISTER_TRANSFORM(TransformTypeLiteral); - REGISTER_TRANSFORM(TransformDefaultExpression); - REGISTER_TRANSFORM(TransformIntervalLiteral); - REGISTER_TRANSFORM(TransformIntervalParameter); - REGISTER_TRANSFORM(TransformSubqueryExpression); - REGISTER_TRANSFORM(TransformMapExpression); - REGISTER_TRANSFORM(TransformMapStructExpression); - REGISTER_TRANSFORM(TransformMapStructField); - REGISTER_TRANSFORM(TransformListComprehensionExpression); - REGISTER_TRANSFORM(TransformListComprehensionFilter); - REGISTER_TRANSFORM(TransformIsDistinctFromOp); - REGISTER_TRANSFORM(TransformGroupingExpression); - REGISTER_TRANSFORM(TransformMethodExpression); - REGISTER_TRANSFORM(TransformRenameList); - REGISTER_TRANSFORM(TransformRenameEntryList); - REGISTER_TRANSFORM(TransformSingleRenameEntry); - REGISTER_TRANSFORM(TransformRenameEntry); - - REGISTER_TRANSFORM(TransformIgnoreOrRespectNulls); } void PEGTransformerFactory::RegisterConnect() { @@ -341,9 +163,6 @@ void PEGTransformerFactory::RegisterSelect() { // select.gram rules that remain manual after generated wrappers are registered. Register("SelectStatementInternal", &TransformSelectStatementInternalRule); REGISTER_TRANSFORM(TransformSimpleSelect); - REGISTER_TRANSFORM(TransformDistinctOrAll); - REGISTER_TRANSFORM(TransformDistinctKeyword); - REGISTER_TRANSFORM(TransformAllKeyword); REGISTER_TRANSFORM(TransformTableRef); REGISTER_TRANSFORM(TransformWithClause); REGISTER_TRANSFORM(TransformWindowDefinition); @@ -371,25 +190,6 @@ void PEGTransformerFactory::RegisterEnums() { RegisterEnum("PlusPrefixOperator", "+"); RegisterEnum("TildePrefixOperator", "~"); - RegisterEnum("OperatorEqual", ExpressionType::COMPARE_EQUAL); - RegisterEnum("OperatorNotEqual", ExpressionType::COMPARE_NOTEQUAL); - RegisterEnum("OperatorLessThan", ExpressionType::COMPARE_LESSTHAN); - RegisterEnum("OperatorGreaterThan", ExpressionType::COMPARE_GREATERTHAN); - RegisterEnum("OperatorLessThanEquals", ExpressionType::COMPARE_LESSTHANOREQUALTO); - RegisterEnum("OperatorGreaterThanEquals", ExpressionType::COMPARE_GREATERTHANOREQUALTO); - - RegisterEnum("TrimBoth", "trim"); - RegisterEnum("TrimLeading", "ltrim"); - RegisterEnum("TrimTrailing", "rtrim"); - - RegisterEnum("LikeToken", "~~"); - RegisterEnum("ILikeToken", "~~*"); - RegisterEnum("GlobToken", "~~~"); - RegisterEnum("SimilarToToken", "regexp_full_match"); - RegisterEnum("NotILikeOp", "!~~*"); - RegisterEnum("NotLikeOp", "!~~"); - RegisterEnum("NotSimilarToOp", "!~"); - RegisterEnum("ExcludeCurrentRow", WindowExcludeMode::CURRENT_ROW); RegisterEnum("ExcludeGroup", WindowExcludeMode::GROUP); RegisterEnum("ExcludeTies", WindowExcludeMode::TIES); @@ -398,9 +198,6 @@ void PEGTransformerFactory::RegisterEnums() { RegisterEnum("SubqueryAny", true); RegisterEnum("SubqueryAll", false); - RegisterEnum("IgnoreNulls", true); - RegisterEnum("RespectNulls", false); - RegisterEnum("IncludeNulls", true); RegisterEnum("ExcludeNulls", false); } diff --git a/src/duckdb/src/parser/peg/transformer/transform_create_trigger.cpp b/src/duckdb/src/parser/peg/transformer/transform_create_trigger.cpp index 39a1e58d8..dcc93b045 100644 --- a/src/duckdb/src/parser/peg/transformer/transform_create_trigger.cpp +++ b/src/duckdb/src/parser/peg/transformer/transform_create_trigger.cpp @@ -51,21 +51,6 @@ Identifier PEGTransformerFactory::TransformTriggerName(PEGTransformer &transform return identifier; } -TriggerForEach PEGTransformerFactory::TransformForEachClause(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.TransformEnum(list_pr.Child(0).GetResult()); -} - -TriggerTiming PEGTransformerFactory::TransformTriggerTiming(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.TransformEnum(list_pr.Child(0).GetResult()); -} - -TriggerEventInfo PEGTransformerFactory::TransformTriggerEvent(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform(list_pr.Child(0).GetResult()); -} - TriggerTiming PEGTransformerFactory::TransformTriggerBefore(PEGTransformer &transformer) { return TriggerTiming::BEFORE; } diff --git a/src/duckdb/src/parser/peg/transformer/transform_expression.cpp b/src/duckdb/src/parser/peg/transformer/transform_expression.cpp index c361e8ccc..e0c6a23c1 100644 --- a/src/duckdb/src/parser/peg/transformer/transform_expression.cpp +++ b/src/duckdb/src/parser/peg/transformer/transform_expression.cpp @@ -73,26 +73,17 @@ PEGTransformerFactory::TransformExpressionStatement(PEGTransformer &transformer, return std::move(select_statement); } -unique_ptr PEGTransformerFactory::TransformExpressionAlias(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(0).GetResult()); -} - -// BaseExpression <- SingleExpression Indirection* -unique_ptr PEGTransformerFactory::TransformBaseExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &indirection_opt = list_pr.Child(1); - if (!indirection_opt.HasResult()) { +unique_ptr +PEGTransformerFactory::TransformBaseExpression(PEGTransformer &transformer, + unique_ptr single_expression, + optional>> indirection_list) { + auto expr = std::move(single_expression); + if (!indirection_list) { return expr; } - auto &indirection_repeat = indirection_opt.GetResult().Cast(); bool prev_indirection_was_cast = false; - for (auto child : indirection_repeat.GetChildren()) { - auto indirection_expr = transformer.Transform>(child); + for (auto &indirection_expr : *indirection_list) { if (indirection_expr->GetExpressionClass() == ExpressionClass::CAST) { auto cast_expr = unique_ptr_cast(std::move(indirection_expr)); cast_expr->ChildMutable() = std::move(expr); @@ -131,30 +122,29 @@ unique_ptr PEGTransformerFactory::TransformBaseExpression(PEGT // ColumnReference <- CatalogReservedSchemaTableColumnName / SchemaReservedTableColumnName / TableReservedColumnName / // NestedColumnName unique_ptr PEGTransformerFactory::TransformColumnReference(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(0).GetResult()); + unique_ptr child) { + return std::move(child); } -unique_ptr -PEGTransformerFactory::TransformCatalogReservedSchemaTableColumnName(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +unique_ptr PEGTransformerFactory::TransformCatalogReservedSchemaTableColumnName( + PEGTransformer &transformer, const Identifier &catalog_qualification, + const Identifier &reserved_schema_qualification, const Identifier &reserved_table_qualification, + const Identifier &reserved_column_name) { vector column_names; - column_names.push_back(transformer.Transform(list_pr.Child(0))); - column_names.push_back(transformer.Transform(list_pr.Child(1))); - column_names.push_back(transformer.Transform(list_pr.Child(2))); - column_names.push_back(list_pr.Child(3).identifier); + column_names.push_back(catalog_qualification); + column_names.push_back(reserved_schema_qualification); + column_names.push_back(reserved_table_qualification); + column_names.push_back(reserved_column_name); return make_uniq(std::move(column_names)); } -unique_ptr -PEGTransformerFactory::TransformSchemaReservedTableColumnName(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +unique_ptr PEGTransformerFactory::TransformSchemaReservedTableColumnName( + PEGTransformer &transformer, const Identifier &schema_qualification, const Identifier &reserved_table_qualification, + const Identifier &reserved_column_name) { vector column_names; - column_names.push_back(transformer.Transform(list_pr.Child(0))); - column_names.push_back(transformer.Transform(list_pr.Child(1))); - column_names.push_back(list_pr.Child(2).identifier); + column_names.push_back(schema_qualification); + column_names.push_back(reserved_table_qualification); + column_names.push_back(reserved_column_name); return make_uniq(std::move(column_names)); } @@ -163,32 +153,22 @@ Identifier PEGTransformerFactory::TransformReservedTableQualification(PEGTransfo return reserved_table_name; } -unique_ptr PEGTransformerFactory::TransformFunctionExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto qualified_function = transformer.Transform(list_pr.Child(0)); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)).Cast(); - bool distinct = false; - transformer.TransformOptional(extract_parens, 0, distinct); - - auto &function_arg_opt = extract_parens.Child(1); - vector function_children; - if (function_arg_opt.HasResult()) { - auto function_argument_list = ExtractParseResultsFromList(function_arg_opt.GetResult()); - for (auto function_argument : function_argument_list) { - function_children.push_back(transformer.Transform(function_argument)); - } - } +unique_ptr PEGTransformerFactory::TransformFunctionExpression( + PEGTransformer &transformer, const QualifiedName &function_identifier, + MethodArguments function_expression_arguments, optional> within_group_clause, + optional> filter_clause, const bool &has_result, + optional> over_clause) { + auto qualified_function = function_identifier; + bool export_clause = has_result; + auto distinct = function_expression_arguments.distinct; + auto function_children = std::move(function_expression_arguments.arguments); auto order_modifier = make_uniq(); - transformer.TransformOptional>(extract_parens, 2, order_modifier->orders); + order_modifier->orders = std::move(function_expression_arguments.order_bys); unique_ptr filter_expr; - transformer.TransformOptional>(list_pr, 3, filter_expr); - bool ignore_nulls = false; - bool has_ignore_nulls_result = extract_parens.Child(3).HasResult(); - transformer.TransformOptional(extract_parens, 3, ignore_nulls); - - auto &export_opt = list_pr.Child(4); + if (filter_clause) { + filter_expr = std::move(*filter_clause); + } if (function_children.size() == 1 && ExpressionIsEmptyStar(*function_children[0].GetExpressionMutable()) && !distinct && order_modifier->orders.empty()) { // COUNT(*) gets converted into COUNT() @@ -196,8 +176,7 @@ unique_ptr PEGTransformerFactory::TransformFunctionExpression( } auto lowercase_name = StringUtil::Lower(qualified_function.name.GetIdentifierName()); - auto &over_opt = list_pr.Child(5); - if (over_opt.HasResult()) { + if (over_clause) { if (transformer.in_window_definition) { throw ParserException("window functions are not allowed in window definitions"); } @@ -207,12 +186,12 @@ unique_ptr PEGTransformerFactory::TransformFunctionExpression( lowercase_name += "_value"; } - if (export_opt.HasResult()) { + if (export_clause) { throw ParserException("EXPORT_STATE is not supported for window functions!"); } transformer.in_window_definition = true; - auto expr = transformer.Transform>(over_opt.GetResult()); + auto expr = std::move(*over_clause); expr->CatalogMutable() = qualified_function.catalog; expr->SchemaMutable() = qualified_function.schema; expr->SetFunctionName(lowercase_name); @@ -221,8 +200,8 @@ unique_ptr PEGTransformerFactory::TransformFunctionExpression( expr->GetArgumentsMutable().push_back(std::move(arg)); } - expr->HasIgnoreNullsMutable() = has_ignore_nulls_result; - expr->IgnoreNullsMutable() = ignore_nulls; + expr->HasIgnoreNullsMutable() = function_expression_arguments.has_ignore_nulls; + expr->IgnoreNullsMutable() = function_expression_arguments.ignore_nulls; expr->FilterMutable() = std::move(filter_expr); expr->ArgOrdersMutable() = std::move(order_modifier->orders); expr->DistinctMutable() = distinct; @@ -310,12 +289,11 @@ unique_ptr PEGTransformerFactory::TransformFunctionExpression( return std::move( make_uniq(LogicalType::DATE, std::move(function_children[0].GetExpressionMutable()))); } - if (has_ignore_nulls_result) { + if (function_expression_arguments.has_ignore_nulls) { throw ParserException("RESPECT/IGNORE NULLS is not supported for non-window functions"); } - auto &within_group_opt = list_pr.Child(2); - if (within_group_opt.HasResult()) { - auto order_by_clause = transformer.Transform>(within_group_opt.GetResult()); + if (within_group_clause) { + auto order_by_clause = std::move(*within_group_clause); if (distinct) { throw ParserException("DISTINCT is not allowed in combination with WITHIN GROUP"); } @@ -350,97 +328,117 @@ unique_ptr PEGTransformerFactory::TransformFunctionExpression( } auto result = make_uniq( qualified_function.catalog, qualified_function.schema, Identifier(lowercase_name), std::move(function_children), - std::move(filter_expr), std::move(order_modifier), distinct, false, export_opt.HasResult()); + std::move(filter_expr), std::move(order_modifier), distinct, false, export_clause); return std::move(result); } +MethodArguments PEGTransformerFactory::TransformFunctionExpressionArgumentList( + PEGTransformer &transformer, const optional &distinct_or_all, + optional> function_argument_list, optional> order_by_clause, + const optional &ignore_or_respect_nulls) { + MethodArguments result; + if (distinct_or_all) { + result.distinct = *distinct_or_all; + } + if (function_argument_list) { + result.arguments = std::move(*function_argument_list); + } + if (order_by_clause) { + result.order_bys = std::move(*order_by_clause); + } + if (ignore_or_respect_nulls) { + result.has_ignore_nulls = true; + result.ignore_nulls = *ignore_or_respect_nulls; + } + return result; +} + vector PEGTransformerFactory::TransformWithinGroupClause(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.GetChild(2)); - return transformer.Transform>(extract_parens); + vector order_by_clause) { + return order_by_clause; +} + +bool PEGTransformerFactory::TransformDistinctKeyword(PEGTransformer &transformer) { + return true; +} + +bool PEGTransformerFactory::TransformAllKeyword(PEGTransformer &transformer) { + return false; } QualifiedName PEGTransformerFactory::TransformFunctionIdentifier(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0); - if (choice_pr.GetResult().type == ParseResultType::IDENTIFIER) { + ParseResult &choice_result) { + if (choice_result.type == ParseResultType::IDENTIFIER) { QualifiedName result; result.catalog = INVALID_CATALOG; result.schema = INVALID_SCHEMA; - result.name = choice_pr.GetResult().Cast().identifier; + result.name = choice_result.Cast().identifier; return result; } - return transformer.Transform(list_pr.Child(0).GetResult()); + return transformer.Transform(choice_result); } QualifiedName PEGTransformerFactory::TransformSchemaReservedFunctionName(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); + const Identifier &schema_qualification, + const Identifier &reserved_function_name) { QualifiedName result; result.catalog = INVALID_CATALOG; - result.schema = Identifier(transformer.Transform(list_pr.Child(0))); - result.name = list_pr.Child(1).identifier; + result.schema = schema_qualification; + result.name = reserved_function_name; return result; } -QualifiedName PEGTransformerFactory::TransformCatalogReservedSchemaFunctionName(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +QualifiedName PEGTransformerFactory::TransformCatalogReservedSchemaFunctionName( + PEGTransformer &transformer, const Identifier &catalog_qualification, + const optional &reserved_schema_qualification, const Identifier &reserved_function_name) { QualifiedName result; - auto &opt_schema = list_pr.Child(1); - if (opt_schema.HasResult()) { - result.catalog = Identifier(transformer.Transform(list_pr.Child(0))); - result.schema = Identifier(transformer.Transform(opt_schema.GetResult())); + if (reserved_schema_qualification) { + result.catalog = catalog_qualification; + result.schema = *reserved_schema_qualification; } else { - result.schema = Identifier(transformer.Transform(list_pr.Child(0))); + result.catalog = INVALID_CATALOG; + result.schema = catalog_qualification; } - result.name = list_pr.Child(2).identifier; + result.name = reserved_function_name; return result; } -unique_ptr PEGTransformerFactory::TransformListExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(0).GetResult()); -} - -unique_ptr PEGTransformerFactory::TransformArrayBoundedListExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - bool is_array = list_pr.Child(0).HasResult(); - auto list_expr = transformer.Transform>>(list_pr.Child(1)); +unique_ptr PEGTransformerFactory::TransformArrayBoundedListExpression( + PEGTransformer &transformer, const bool &has_result, vector> bounded_list_expression) { + bool is_array = has_result; if (!is_array) { - return make_uniq(INVALID_CATALOG, "main", "list_value", std::move(list_expr)); + return make_uniq(INVALID_CATALOG, "main", "list_value", std::move(bounded_list_expression)); } - return make_uniq(ExpressionType::ARRAY_CONSTRUCTOR, std::move(list_expr)); + return make_uniq(ExpressionType::ARRAY_CONSTRUCTOR, std::move(bounded_list_expression)); } -unique_ptr PEGTransformerFactory::TransformFilterClause(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); - auto &inner_list = extract_parens.Cast(); - return transformer.Transform>(inner_list.Child(1)); +unique_ptr +PEGTransformerFactory::TransformFilterClause(PEGTransformer &transformer, + unique_ptr filter_clause_expression) { + return filter_clause_expression; } -unique_ptr PEGTransformerFactory::TransformParenthesisExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - // ParenthesisExpression <- Parens(List(Expression)) - vector> children; +unique_ptr +PEGTransformerFactory::TransformFilterClauseExpression(PEGTransformer &transformer, + unique_ptr filter_clause_contents) { + return filter_clause_contents; +} - auto &list_pr = parse_result.Cast(); - auto expressions = ExtractParseResultsFromList(ExtractResultFromParens(list_pr.Child(0))); +unique_ptr +PEGTransformerFactory::TransformFilterClauseContents(PEGTransformer &transformer, const bool &has_result, + unique_ptr expression) { + return expression; +} - for (auto &expression : expressions) { - children.push_back(transformer.Transform>(expression)); - } - if (children.size() == 1) { - return std::move(children[0]); +unique_ptr +PEGTransformerFactory::TransformParenthesisExpression(PEGTransformer &transformer, + vector> expression) { + // ParenthesisExpression <- Parens(List(Expression)) + if (expression.size() == 1) { + return std::move(expression[0]); } - return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "row", std::move(children)); + return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "row", std::move(expression)); } void PEGTransformerFactory::RemoveOrderQualificationRecursive(unique_ptr &root_expr) { @@ -453,12 +451,11 @@ void PEGTransformerFactory::RemoveOrderQualificationRecursive(unique_ptr PEGTransformerFactory::TransformArrayParensSelect(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); +unique_ptr +PEGTransformerFactory::TransformArrayParensSelect(PEGTransformer &transformer, + unique_ptr select_statement_internal) { auto subquery_expr = make_uniq(); - subquery_expr->SubqueryMutable() = transformer.Transform>(extract_parens); + subquery_expr->SubqueryMutable() = std::move(select_statement_internal); // ARRAY expression // wrap subquery into // "SELECT CASE WHEN ARRAY_AGG(col) IS NULL THEN [] ELSE ARRAY_AGG(col) END FROM (...) tbl" @@ -545,39 +542,27 @@ unique_ptr PEGTransformerFactory::TransformArrayParensSelect(P } unique_ptr PEGTransformerFactory::TransformStructExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); + vector struct_field) { auto func_name = "struct_pack"; - vector struct_children; - auto struct_children_list = ExtractParseResultsFromList(list_pr.GetChild(1)); - for (auto struct_child : struct_children_list) { - struct_children.push_back(transformer.Transform(struct_child)); - } - - return make_uniq(INVALID_CATALOG, "main", func_name, std::move(struct_children)); + return make_uniq(INVALID_CATALOG, "main", func_name, std::move(struct_field)); } -FunctionArgument PEGTransformerFactory::TransformStructField(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto alias = transformer.Transform(list_pr.Child(0)); - auto expr = transformer.Transform>(list_pr.Child(2)); - expr->SetAlias(Identifier(alias)); +FunctionArgument PEGTransformerFactory::TransformStructField(PEGTransformer &transformer, + const Identifier &col_id_or_string, + unique_ptr expression) { + auto alias = col_id_or_string.GetIdentifierName(); + expression->SetAlias(Identifier(alias)); - return FunctionArgument(Identifier(std::move(alias)), std::move(expr)); + return FunctionArgument(Identifier(std::move(alias)), std::move(expression)); } -vector> PEGTransformerFactory::TransformBoundedListExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &has_expr = list_pr.Child(1); - vector> list_children; - if (has_expr.HasResult()) { - auto expr_list = ExtractParseResultsFromList(has_expr.GetResult()); - for (auto &expr : expr_list) { - list_children.push_back(transformer.Transform>(expr)); - } +vector> +PEGTransformerFactory::TransformBoundedListExpression(PEGTransformer &transformer, + optional>> expression) { + if (expression) { + return std::move(*expression); } - return list_children; + return vector>(); } // Expression <- LambdaArrowExpression @@ -588,75 +573,70 @@ unique_ptr PEGTransformerFactory::TransformExpression(PEGTrans return transformer.Transform>(list_pr.Child(0)); } -unique_ptr PEGTransformerFactory::TransformLambdaArrowExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &lambda_opt = list_pr.Child(1); - if (!lambda_opt.HasResult()) { +unique_ptr PEGTransformerFactory::TransformLambdaArrowExpression( + PEGTransformer &transformer, unique_ptr logical_or_expression, + optional>> single_arrow_pair) { + auto expr = std::move(logical_or_expression); + if (!single_arrow_pair) { return expr; } - // Each child is a SingleArrowPair ListParseResult: ['->', LogicalOrExpression] - auto &inner_lambda_list = lambda_opt.GetResult().Cast(); - for (auto pair_node : inner_lambda_list.GetChildren()) { - auto &pair_list = pair_node.get().Cast(); // SingleArrowPair - auto right_expr = transformer.Transform>(pair_list.Child(1)); + for (auto &right_expr : *single_arrow_pair) { expr = make_uniq(std::move(expr), std::move(right_expr)); } return expr; } -// LogicalOrExpression <- LogicalAndExpression ('OR' LogicalAndExpression)* -unique_ptr PEGTransformerFactory::TransformLogicalOrExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &or_expr_opt = list_pr.Child(1); - if (!or_expr_opt.HasResult()) { +static unique_ptr FoldConjunctionExpression(PEGTransformer &transformer, + unique_ptr expression, + optional>> tails, + ExpressionType conjunction_type) { + auto expr = std::move(expression); + if (!tails) { return expr; } - auto &or_expr_repeat = or_expr_opt.GetResult().Cast(); - auto or_depth_guard = transformer.StackCheck(or_expr_repeat.GetChildren().size()); - for (auto &or_expr : or_expr_repeat.GetChildren()) { - auto &inner_list_pr = or_expr.get().Cast(); - auto right_expr = transformer.Transform>(inner_list_pr.Child(1)); - expr = make_uniq(ExpressionType::CONJUNCTION_OR, std::move(expr), std::move(right_expr)); + auto depth_guard = transformer.StackCheck(tails->size()); + for (auto &tail : *tails) { + expr = make_uniq(conjunction_type, std::move(expr), std::move(tail)); } return expr; } -// LogicalAndExpression <- LogicalNotExpression ('AND' LogicalNotExpression)* -unique_ptr PEGTransformerFactory::TransformLogicalAndExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &and_expr_opt = list_pr.Child(1); - if (!and_expr_opt.HasResult()) { - return expr; - } - auto &and_expr_repeat = and_expr_opt.GetResult().Cast(); - auto and_depth_guard = transformer.StackCheck(and_expr_repeat.GetChildren().size()); - for (auto &and_expr : and_expr_repeat.GetChildren()) { - auto &inner_list_pr = and_expr.get().Cast(); - auto right_expr = transformer.Transform>(inner_list_pr.Child(1)); - expr = - make_uniq(ExpressionType::CONJUNCTION_AND, std::move(expr), std::move(right_expr)); - } - return expr; +unique_ptr PEGTransformerFactory::TransformLogicalOrExpression( + PEGTransformer &transformer, unique_ptr logical_and_expression, + optional>> logical_or_expression_tail) { + return FoldConjunctionExpression(transformer, std::move(logical_and_expression), + std::move(logical_or_expression_tail), ExpressionType::CONJUNCTION_OR); } -// LogicalNotExpression <- 'NOT'* IsExpression -unique_ptr PEGTransformerFactory::TransformLogicalNotExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(1)); - auto ¬_expr_opt = list_pr.Child(0); - if (!not_expr_opt.HasResult()) { +unique_ptr PEGTransformerFactory::TransformLogicalAndExpression( + PEGTransformer &transformer, unique_ptr logical_not_expression, + optional>> logical_and_expression_tail) { + return FoldConjunctionExpression(transformer, std::move(logical_not_expression), + std::move(logical_and_expression_tail), ExpressionType::CONJUNCTION_AND); +} + +unique_ptr PEGTransformerFactory::TransformColDefOrExpr( + PEGTransformer &transformer, unique_ptr col_def_and_expr, + optional>> col_def_or_expression_tail) { + return FoldConjunctionExpression(transformer, std::move(col_def_and_expr), std::move(col_def_or_expression_tail), + ExpressionType::CONJUNCTION_OR); +} + +unique_ptr PEGTransformerFactory::TransformColDefAndExpr( + PEGTransformer &transformer, unique_ptr is_distinct_from_expression, + optional>> col_def_and_expression_tail) { + return FoldConjunctionExpression(transformer, std::move(is_distinct_from_expression), + std::move(col_def_and_expression_tail), ExpressionType::CONJUNCTION_AND); +} + +unique_ptr +PEGTransformerFactory::TransformLogicalNotExpression(PEGTransformer &transformer, optional> not_expression, + unique_ptr is_expression) { + auto expr = std::move(is_expression); + if (!not_expression) { return expr; } - auto ¬_expr_repeat = not_expr_opt.GetResult().Cast(); - size_t n = not_expr_repeat.GetChildren().size(); - for (size_t i = 0; i < n; i++) { + for (idx_t i = 0; i < not_expression->size(); i++) { vector> inner_list_children; inner_list_children.push_back(std::move(expr)); expr = make_uniq(ExpressionType::OPERATOR_NOT, std::move(inner_list_children)); @@ -664,18 +644,15 @@ unique_ptr PEGTransformerFactory::TransformLogicalNotExpressio return expr; } -// IsExpression <- IsDistinctFromExpression IsTest* -unique_ptr PEGTransformerFactory::TransformIsExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &is_test_opt = list_pr.Child(1); - if (!is_test_opt.HasResult()) { +unique_ptr +PEGTransformerFactory::TransformIsExpression(PEGTransformer &transformer, + unique_ptr is_distinct_from_expression, + optional>> is_test) { + auto expr = std::move(is_distinct_from_expression); + if (!is_test) { return expr; } - auto &is_test_expr_repeat = is_test_opt.GetResult().Cast(); - for (auto &is_test_expr : is_test_expr_repeat.GetChildren()) { - auto is_expr = transformer.Transform>(is_test_expr); + for (auto &is_expr : *is_test) { if (is_expr->GetExpressionClass() == ExpressionClass::COMPARISON) { auto compare_expr = unique_ptr_cast(std::move(is_expr)); compare_expr->LeftMutable() = make_uniq(LogicalType::BOOLEAN, std::move(expr)); @@ -692,91 +669,110 @@ unique_ptr PEGTransformerFactory::TransformIsExpression(PEGTra return expr; } -unique_ptr PEGTransformerFactory::TransformIsTest(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(0).GetResult()); -} - unique_ptr PEGTransformerFactory::TransformIsLiteral(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto ¬_expr = list_pr.Child(1); - auto &inner_list_pr = list_pr.Child(2); - auto literal_value = transformer.Transform(inner_list_pr.Child(0).GetResult()); - if (literal_value.IsNull()) { - auto expr_type = not_expr.HasResult() ? ExpressionType::OPERATOR_IS_NOT_NULL : ExpressionType::OPERATOR_IS_NULL; + const bool &has_result, + const Value &is_literal_value) { + if (is_literal_value.IsNull()) { + auto expr_type = has_result ? ExpressionType::OPERATOR_IS_NOT_NULL : ExpressionType::OPERATOR_IS_NULL; return make_uniq(expr_type, nullptr); } - auto expr_type = - not_expr.HasResult() ? ExpressionType::COMPARE_DISTINCT_FROM : ExpressionType::COMPARE_NOT_DISTINCT_FROM; - return make_uniq(expr_type, nullptr, make_uniq(literal_value)); + auto expr_type = has_result ? ExpressionType::COMPARE_DISTINCT_FROM : ExpressionType::COMPARE_NOT_DISTINCT_FROM; + return make_uniq(expr_type, nullptr, make_uniq(is_literal_value)); } -unique_ptr PEGTransformerFactory::TransformNotNull(PEGTransformer &transformer, - ParseResult &parse_result) { +unique_ptr PEGTransformerFactory::TransformNotNullKeyword(PEGTransformer &transformer) { return make_uniq(ExpressionType::OPERATOR_IS_NOT_NULL, nullptr); } -unique_ptr PEGTransformerFactory::TransformIsNull(PEGTransformer &transformer, - ParseResult &parse_result) { +unique_ptr PEGTransformerFactory::TransformNotNullOperator(PEGTransformer &transformer) { + return make_uniq(ExpressionType::OPERATOR_IS_NOT_NULL, nullptr); +} + +unique_ptr PEGTransformerFactory::TransformIsNullOperator(PEGTransformer &transformer) { return make_uniq(ExpressionType::OPERATOR_IS_NULL, nullptr); } -// IsDistinctFromExpression <- ComparisonExpression (IsDistinctFromOp ComparisonExpression)* -unique_ptr PEGTransformerFactory::TransformIsDistinctFromExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &is_test_opt = list_pr.Child(1); - if (!is_test_opt.HasResult()) { +bool PEGTransformerFactory::TransformNotKeyword(PEGTransformer &transformer) { + return true; +} + +unique_ptr +PEGTransformerFactory::TransformIsDistinctFromExpression(PEGTransformer &transformer, + unique_ptr comparison_expression, + optional> is_distinct_from_tail) { + auto expr = std::move(comparison_expression); + if (!is_distinct_from_tail) { return expr; } - auto &is_distinct_repeat = is_test_opt.GetResult().Cast(); - for (auto &is_distinct : is_distinct_repeat.GetChildren()) { - auto &distinct_list = is_distinct.get().Cast(); - auto distinct_type = transformer.Transform(distinct_list.Child(0)); - auto right_expr = transformer.Transform>(distinct_list.Child(1)); - auto distinct_operator = make_uniq(distinct_type, std::move(expr), std::move(right_expr)); + for (auto &is_distinct : *is_distinct_from_tail) { + auto distinct_operator = make_uniq(is_distinct.comparison_type, std::move(expr), + std::move(is_distinct.expression)); expr = std::move(distinct_operator); } return expr; } -// ComparisonExpression <- BetweenInLikeExpression (ComparisonOperator 'NOT'* BetweenInLikeExpression)* -unique_ptr PEGTransformerFactory::TransformComparisonExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &comparison_opt = list_pr.Child(1); - if (!comparison_opt.HasResult()) { +unique_ptr PEGTransformerFactory::TransformComparisonExpression( + PEGTransformer &transformer, unique_ptr between_in_like_expression, + optional> comparison_expression_tail) { + auto expr = std::move(between_in_like_expression); + if (!comparison_expression_tail) { return expr; } - auto &comparison_repeat = comparison_opt.GetResult().Cast(); - auto cmp_depth_guard = transformer.StackCheck(comparison_repeat.GetChildren().size()); - for (auto &comparison_expr : comparison_repeat.GetChildren()) { - auto &inner_list_pr = comparison_expr.get().Cast(); - auto comparison_operator = transformer.Transform(inner_list_pr.Child(0)); - auto ¬_expr_opt = inner_list_pr.Child(1); - auto right_expr = transformer.Transform>(inner_list_pr.Child(2)); - if (not_expr_opt.HasResult()) { - auto ¬_expr_repeat = not_expr_opt.GetResult().Cast(); - for (size_t i = 0; i < not_expr_repeat.GetChildren().size(); i++) { - vector> inner_list_children; - inner_list_children.push_back(std::move(right_expr)); - right_expr = - make_uniq(ExpressionType::OPERATOR_NOT, std::move(inner_list_children)); - } + auto cmp_depth_guard = transformer.StackCheck(comparison_expression_tail->size()); + for (auto &comparison_expr : *comparison_expression_tail) { + auto right_expr = std::move(comparison_expr.expression); + for (idx_t i = 0; i < comparison_expr.not_keywords.size(); i++) { + vector> inner_list_children; + inner_list_children.push_back(std::move(right_expr)); + right_expr = make_uniq(ExpressionType::OPERATOR_NOT, std::move(inner_list_children)); } - expr = make_uniq(comparison_operator, std::move(expr), std::move(right_expr)); + expr = make_uniq(comparison_expr.comparison_type, std::move(expr), std::move(right_expr)); } return expr; } -ExpressionType PEGTransformerFactory::TransformComparisonOperator(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.TransformEnum(list_pr.Child(0).GetResult()); +IsDistinctFromTail +PEGTransformerFactory::TransformIsDistinctFromTail(PEGTransformer &transformer, + const ExpressionType &is_distinct_from_op, + unique_ptr comparison_expression) { + return {is_distinct_from_op, std::move(comparison_expression)}; +} + +ComparisonExpressionTail PEGTransformerFactory::TransformComparisonExpressionTail( + PEGTransformer &transformer, const ExpressionType &comparison_operator, optional> not_expression, + unique_ptr between_in_like_expression) { + ComparisonExpressionTail result; + result.comparison_type = comparison_operator; + if (not_expression) { + result.not_keywords = std::move(*not_expression); + } + result.expression = std::move(between_in_like_expression); + return result; +} + +ExpressionType PEGTransformerFactory::TransformOperatorEqual(PEGTransformer &transformer) { + return ExpressionType::COMPARE_EQUAL; +} + +ExpressionType PEGTransformerFactory::TransformOperatorNotEqual(PEGTransformer &transformer) { + return ExpressionType::COMPARE_NOTEQUAL; +} + +ExpressionType PEGTransformerFactory::TransformOperatorLessThan(PEGTransformer &transformer) { + return ExpressionType::COMPARE_LESSTHAN; +} + +ExpressionType PEGTransformerFactory::TransformOperatorGreaterThan(PEGTransformer &transformer) { + return ExpressionType::COMPARE_GREATERTHAN; +} + +ExpressionType PEGTransformerFactory::TransformOperatorLessThanEquals(PEGTransformer &transformer) { + return ExpressionType::COMPARE_LESSTHANOREQUALTO; +} + +ExpressionType PEGTransformerFactory::TransformOperatorGreaterThanEquals(PEGTransformer &transformer) { + return ExpressionType::COMPARE_GREATERTHANOREQUALTO; } bool TryNegateLikeFunction(Identifier &function_name) { @@ -794,18 +790,16 @@ bool TryNegateLikeFunction(Identifier &function_name) { return false; } -// BetweenInLikeExpression <- OtherOperatorExpression BetweenInLikeOp? -unique_ptr PEGTransformerFactory::TransformBetweenInLikeExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &between_in_like_opt = list_pr.Child(1); - if (!between_in_like_opt.HasResult()) { +unique_ptr +PEGTransformerFactory::TransformBetweenInLikeExpression(PEGTransformer &transformer, + unique_ptr other_operator_expression, + optional between_in_like_op) { + auto expr = std::move(other_operator_expression); + if (!between_in_like_op) { return expr; } - auto between_in_like_expr = transformer.Transform>(between_in_like_opt.GetResult()); - auto &op_list = between_in_like_opt.GetResult().Cast(); - bool has_not = op_list.Child(0).HasResult(); + auto between_in_like_expr = std::move(between_in_like_op->expression); + bool has_not = between_in_like_op->has_not; if (between_in_like_expr->GetExpressionClass() == ExpressionClass::BETWEEN) { auto between_expr = unique_ptr_cast(std::move(between_in_like_expr)); between_expr->InputMutable() = std::move(expr); @@ -855,43 +849,32 @@ unique_ptr PEGTransformerFactory::TransformBetweenInLikeExpres return expr; } -unique_ptr PEGTransformerFactory::TransformBetweenInLikeOp(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &inner_list = list_pr.Child(1); - auto expr = transformer.Transform>(inner_list.Child(0).GetResult()); - return expr; +BetweenInLikeOperator +PEGTransformerFactory::TransformBetweenInLikeOp(PEGTransformer &transformer, const bool &has_result, + unique_ptr between_in_like_op_expression) { + BetweenInLikeOperator result; + result.has_not = has_result; + result.expression = std::move(between_in_like_op_expression); + return result; } unique_ptr PEGTransformerFactory::TransformInClause(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(1)); + unique_ptr in_expression) { + return in_expression; } -unique_ptr PEGTransformerFactory::TransformInExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - if (StringUtil::CIEquals(choice_pr.name, "InExpressionList") || - StringUtil::CIEquals(choice_pr.name, "InSelectStatement")) { - return transformer.Transform>(choice_pr); - } - auto right_expr = transformer.Transform>(choice_pr); +unique_ptr +PEGTransformerFactory::TransformInContainsExpression(PEGTransformer &transformer, + unique_ptr other_operator_expression) { vector> children; - children.push_back(std::move(right_expr)); + children.push_back(std::move(other_operator_expression)); return make_uniq("contains", std::move(children)); } -unique_ptr PEGTransformerFactory::TransformInExpressionList(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(0)); - auto expr_list_pr = ExtractParseResultsFromList(extract_parens); - vector> in_children; - for (auto &expr : expr_list_pr) { - in_children.push_back(transformer.Transform>(expr)); - } +unique_ptr +PEGTransformerFactory::TransformInExpressionList(PEGTransformer &transformer, + vector> expression) { + auto in_children = std::move(expression); if (in_children.size() == 1 && in_children[0]->GetExpressionClass() == ExpressionClass::SUBQUERY) { auto &subquery_expr = in_children[0]->Cast(); auto result = make_uniq(); @@ -904,40 +887,39 @@ unique_ptr PEGTransformerFactory::TransformInExpressionList(PE return std::move(result); } -unique_ptr PEGTransformerFactory::TransformInSelectStatement(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(0)); +unique_ptr +PEGTransformerFactory::TransformInSelectStatement(PEGTransformer &transformer, + unique_ptr select_statement_internal) { auto result = make_uniq(); result->GetSubqueryTypeMutable() = SubqueryType::ANY; result->GetComparisonTypeMutable() = ExpressionType::COMPARE_EQUAL; - result->SubqueryMutable() = transformer.Transform>(extract_parens); + result->SubqueryMutable() = std::move(select_statement_internal); return std::move(result); } -unique_ptr PEGTransformerFactory::TransformBetweenClause(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto lower = transformer.Transform>(list_pr.Child(1)); - auto higher = transformer.Transform>(list_pr.Child(3)); - auto result = make_uniq(nullptr, std::move(lower), std::move(higher)); +unique_ptr +PEGTransformerFactory::TransformBetweenClause(PEGTransformer &transformer, + unique_ptr other_operator_expression, + unique_ptr other_operator_expression_1) { + auto result = make_uniq(nullptr, std::move(other_operator_expression), + std::move(other_operator_expression_1)); return std::move(result); } -unique_ptr PEGTransformerFactory::TransformLikeClause(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - string like_variation = transformer.Transform(list_pr.Child(0)); +unique_ptr +PEGTransformerFactory::TransformLikeClause(PEGTransformer &transformer, const string &like_variations, + unique_ptr other_operator_expression, + optional> escape_clause) { + string like_variation = like_variations; vector> like_children; - like_children.push_back(transformer.Transform>(list_pr.Child(1))); - auto &escape_opt = list_pr.Child(2); - if (escape_opt.HasResult()) { + like_children.push_back(std::move(other_operator_expression)); + if (escape_clause) { if (like_variation == "~~") { like_variation = "like_escape"; } else if (like_variation == "~~*") { like_variation = "ilike_escape"; } - like_children.push_back(transformer.Transform>(escape_opt.GetResult())); + like_children.push_back(std::move(*escape_clause)); } auto result = make_uniq(Identifier(like_variation), std::move(like_children)); if (like_variation != "regexp_full_match") { @@ -946,36 +928,53 @@ unique_ptr PEGTransformerFactory::TransformLikeClause(PEGTrans return std::move(result); } -unique_ptr PEGTransformerFactory::TransformEscapeClause(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.GetChild(1)); +unique_ptr +PEGTransformerFactory::TransformEscapeClause(PEGTransformer &transformer, + unique_ptr comparison_expression) { + return comparison_expression; } -string PEGTransformerFactory::TransformLikeVariations(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.TransformEnum(list_pr.Child(0).GetResult()); +string PEGTransformerFactory::TransformLikeToken(PEGTransformer &transformer) { + return "~~"; } -// OtherOperatorExpression <- BitwiseExpression (OtherOperator BitwiseExpression)* -unique_ptr PEGTransformerFactory::TransformOtherOperatorExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &other_operator_opt = list_pr.Child(1); - if (!other_operator_opt.HasResult()) { +string PEGTransformerFactory::TransformILikeToken(PEGTransformer &transformer) { + return "~~*"; +} + +string PEGTransformerFactory::TransformGlobToken(PEGTransformer &transformer) { + return "~~~"; +} + +string PEGTransformerFactory::TransformSimilarToToken(PEGTransformer &transformer) { + return "regexp_full_match"; +} + +string PEGTransformerFactory::TransformNotILikeOp(PEGTransformer &transformer) { + return "!~~*"; +} + +string PEGTransformerFactory::TransformNotLikeOp(PEGTransformer &transformer) { + return "!~~"; +} + +string PEGTransformerFactory::TransformNotSimilarToOp(PEGTransformer &transformer) { + return "!~"; +} + +unique_ptr +PEGTransformerFactory::TransformOtherOperatorExpression(PEGTransformer &transformer, + unique_ptr bitwise_expression, + optional> other_operator_tail) { + auto expr = std::move(bitwise_expression); + if (!other_operator_tail) { return expr; } - auto &other_operator_repeat = other_operator_opt.GetResult().Cast(); - for (auto &other_operator_expr : other_operator_repeat.GetChildren()) { - auto &inner_list_pr = other_operator_expr.get().Cast(); - auto right_expr = transformer.Transform>(inner_list_pr.Child(1)); - auto &other_operator_pr = inner_list_pr.Child(0); - auto &other_operator_choice = other_operator_pr.Child(0).GetResult(); - if (StringUtil::CIEquals(other_operator_choice.name, "AnyAllOperator")) { - auto any_all = transformer.Transform>(other_operator_choice); - auto op_string = any_all.first; - auto is_any = any_all.second; + for (auto &other_operator_expr : *other_operator_tail) { + auto right_expr = std::move(other_operator_expr.expression); + if (other_operator_expr.op.is_any_all) { + auto op_string = other_operator_expr.op.name; + auto is_any = other_operator_expr.op.is_any; // Map operator string to ExpressionType (INVALID if not a comparison operator) auto expression_type = OperatorToExpressionType(op_string); @@ -1028,7 +1027,7 @@ unique_ptr PEGTransformerFactory::TransformOtherOperatorExpres return std::move(subquery_expr); } } else { - auto other_operator = transformer.Transform(other_operator_pr); + auto other_operator = std::move(other_operator_expr.op.name); vector> children_function; children_function.push_back(std::move(expr)); children_function.push_back(std::move(right_expr)); @@ -1054,161 +1053,121 @@ unique_ptr PEGTransformerFactory::TransformOtherOperatorExpres return expr; } -string PEGTransformerFactory::TransformOtherOperator(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &child = list_pr.Child(0).GetResult(); - // OperatorLiteral matches any operator token and produces an OperatorParseResult directly - if (child.type == ParseResultType::OPERATOR) { - return child.Cast().operator_token; - } - return transformer.Transform(child); +OtherOperatorTail PEGTransformerFactory::TransformOtherOperatorTail(PEGTransformer &transformer, + ParsedOperator other_operator, + unique_ptr bitwise_expression) { + OtherOperatorTail result; + result.op = std::move(other_operator); + result.expression = std::move(bitwise_expression); + return result; } -// QualifiedOperator <- 'OPERATOR' Parens(ColId* AnyOp) -string PEGTransformerFactory::TransformQualifiedOperator(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &any_op_pr = ExtractResultFromParens(list_pr.GetChild(1)).Cast(); - auto &repeat_colid_opt = any_op_pr.Child(0); - vector result; - if (repeat_colid_opt.HasResult()) { - auto &repeat_colid = repeat_colid_opt.GetResult().Cast(); - for (auto &colid : repeat_colid.GetChildren()) { - result.push_back(transformer.Transform(colid)); - } +ParsedOperator PEGTransformerFactory::TransformOtherOperator(PEGTransformer &transformer, ParseResult &choice_result) { + ParsedOperator result; + // OperatorLiteral matches any operator token and produces an OperatorParseResult directly + if (choice_result.type == ParseResultType::OPERATOR) { + result.name = choice_result.Cast().operator_token; + return result; } - result.push_back(transformer.Transform(any_op_pr.GetChild(1))); - return StringUtil::Join(result, "."); -} - -// AnyOp <- '!~~*' / '>>=' / ... / '!' -string PEGTransformerFactory::TransformAnyOp(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - return choice_pr.Cast().keyword; -} - -string PEGTransformerFactory::TransformJsonOperator(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return list_pr.Child(0).keyword; + if (StringUtil::CIEquals(choice_result.name, "AnyAllOperator")) { + auto any_all = transformer.Transform>(choice_result); + result.name = any_all.first; + result.is_any_all = true; + result.is_any = any_all.second; + return result; + } + result.name = transformer.Transform(choice_result); + return result; } -string PEGTransformerFactory::TransformInetOperator(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - return choice_pr.Cast().keyword; +string PEGTransformerFactory::TransformQualifiedOperator(PEGTransformer &transformer, + const string &qualified_operator_contents) { + return qualified_operator_contents; } -string PEGTransformerFactory::TransformStringOperator(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - return choice_pr.Cast().keyword; +string PEGTransformerFactory::TransformQualifiedOperatorContents(PEGTransformer &transformer, + const optional> &col_id_dot, + const string &any_op) { + vector result; + if (col_id_dot) { + result = *col_id_dot; + } + result.push_back(any_op); + return StringUtil::Join(result, "."); } -string PEGTransformerFactory::TransformListOperator(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - return choice_pr.Cast().keyword; +pair PEGTransformerFactory::TransformAnyAllOperator(PEGTransformer &transformer, const string &any_op, + const bool &any_or_all) { + return make_pair(any_op, any_or_all); } -pair PEGTransformerFactory::TransformAnyAllOperator(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto op_string = transformer.Transform(list_pr.Child(0)); - auto subquery_type = transformer.Transform(list_pr.Child(1)); - return make_pair(op_string, subquery_type); +bool PEGTransformerFactory::TransformSubqueryAny(PEGTransformer &transformer) { + return true; } -bool PEGTransformerFactory::TransformAnyOrAll(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.TransformEnum(list_pr.Child(0).GetResult()); +bool PEGTransformerFactory::TransformSubqueryAll(PEGTransformer &transformer) { + return false; } -// BitwiseExpression <- AdditiveExpression (BitOperator AdditiveExpression)* -unique_ptr PEGTransformerFactory::TransformBitwiseExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &bit_operator_opt = list_pr.Child(1); - if (!bit_operator_opt.HasResult()) { +unique_ptr +PEGTransformerFactory::TransformBitwiseExpression(PEGTransformer &transformer, + unique_ptr additive_expression, + optional> bitwise_expression_tail) { + auto expr = std::move(additive_expression); + if (!bitwise_expression_tail) { return expr; } - auto &bit_repeat = bit_operator_opt.GetResult().Cast(); - auto bit_depth_guard = transformer.StackCheck(bit_repeat.GetChildren().size()); - for (auto &bit_expr : bit_repeat.GetChildren()) { - auto &inner_list_pr = bit_expr.get().Cast(); - auto bit = transformer.Transform(inner_list_pr.Child(0)); + auto bit_depth_guard = transformer.StackCheck(bitwise_expression_tail->size()); + for (auto &bit_expr : *bitwise_expression_tail) { vector> bit_children; bit_children.push_back(std::move(expr)); - bit_children.push_back( - transformer.Transform>(inner_list_pr.Child(1))); - auto func_expr = make_uniq(Identifier(std::move(bit)), std::move(bit_children)); + bit_children.push_back(std::move(bit_expr.expression)); + auto func_expr = make_uniq(Identifier(std::move(bit_expr.op)), std::move(bit_children)); func_expr->IsOperatorMutable() = true; expr = std::move(func_expr); } return expr; } -string PEGTransformerFactory::TransformBitOperator(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - return choice_pr.Cast().keyword; -} - -// AdditiveExpression <- MultiplicativeExpression (Term MultiplicativeExpression)* -unique_ptr PEGTransformerFactory::TransformAdditiveExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &term_opt = list_pr.Child(1); - if (!term_opt.HasResult()) { +unique_ptr +PEGTransformerFactory::TransformAdditiveExpression(PEGTransformer &transformer, + unique_ptr multiplicative_expression, + optional> additive_expression_tail) { + auto expr = std::move(multiplicative_expression); + if (!additive_expression_tail) { return expr; } - auto &term_repeat = term_opt.GetResult().Cast(); - auto add_depth_guard = transformer.StackCheck(term_repeat.GetChildren().size()); - for (auto &term_expr : term_repeat.GetChildren()) { - auto &inner_list_pr = term_expr.get().Cast(); - auto term = transformer.Transform(inner_list_pr.Child(0)); + auto add_depth_guard = transformer.StackCheck(additive_expression_tail->size()); + for (auto &term_expr : *additive_expression_tail) { vector> term_children; term_children.push_back(std::move(expr)); - term_children.push_back( - transformer.Transform>(inner_list_pr.Child(1))); - auto func_expr = make_uniq(Identifier(std::move(term)), std::move(term_children)); + term_children.push_back(std::move(term_expr.expression)); + auto func_expr = make_uniq(Identifier(std::move(term_expr.op)), std::move(term_children)); func_expr->IsOperatorMutable() = true; - if (inner_list_pr.offset.IsValid()) { - transformer.SetQueryLocation(*func_expr, inner_list_pr.offset); + if (term_expr.query_location.IsValid()) { + transformer.SetQueryLocation(*func_expr, term_expr.query_location); } expr = std::move(func_expr); } return expr; } -string PEGTransformerFactory::TransformTerm(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - return choice_pr.Cast().keyword; -} - -// MultiplicativeExpression <- ExponentiationExpression (Factor ExponentiationExpression)* -unique_ptr PEGTransformerFactory::TransformMultiplicativeExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &factor_opt = list_pr.Child(1); - if (!factor_opt.HasResult()) { +unique_ptr PEGTransformerFactory::TransformMultiplicativeExpression( + PEGTransformer &transformer, unique_ptr exponentiation_expression, + optional> multiplicative_expression_tail) { + auto expr = std::move(exponentiation_expression); + if (!multiplicative_expression_tail) { return expr; } - auto &factor_repeat = factor_opt.GetResult().Cast(); - auto mul_depth_guard = transformer.StackCheck(factor_repeat.GetChildren().size()); - for (auto &factor_expr : factor_repeat.GetChildren()) { - auto &inner_list_pr = factor_expr.get().Cast(); - auto factor = transformer.Transform(inner_list_pr.Child(0)); + auto mul_depth_guard = transformer.StackCheck(multiplicative_expression_tail->size()); + for (auto &factor_expr : *multiplicative_expression_tail) { + auto factor = std::move(factor_expr.op); if (factor == "/" && transformer.options.integer_division) { factor = "//"; } vector> factor_children; factor_children.push_back(std::move(expr)); - factor_children.push_back( - transformer.Transform>(inner_list_pr.Child(1))); + factor_children.push_back(std::move(factor_expr.expression)); auto func_expr = make_uniq(Identifier(std::move(factor)), std::move(factor_children)); func_expr->IsOperatorMutable() = true; expr = std::move(func_expr); @@ -1216,57 +1175,57 @@ unique_ptr PEGTransformerFactory::TransformMultiplicativeExpre return expr; } -string PEGTransformerFactory::TransformFactor(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - return choice_pr.Cast().keyword; -} - -// ExponentiationExpression <- CollateExpression (ExponentOperator CollateExpression)* -unique_ptr PEGTransformerFactory::TransformExponentiationExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &exponent_opt = list_pr.Child(1); - if (!exponent_opt.HasResult()) { +unique_ptr PEGTransformerFactory::TransformExponentiationExpression( + PEGTransformer &transformer, unique_ptr collate_expression, + optional> exponentiation_expression_tail) { + auto expr = std::move(collate_expression); + if (!exponentiation_expression_tail) { return expr; } - auto &exponent_repeat = exponent_opt.GetResult().Cast(); - for (auto &exponent_expr : exponent_repeat.GetChildren()) { - auto &inner_list_pr = exponent_expr.get().Cast(); - auto exponent = transformer.Transform(inner_list_pr.Child(0)); + for (auto &exponent_expr : *exponentiation_expression_tail) { vector> exponent_children; exponent_children.push_back(std::move(expr)); - exponent_children.push_back( - transformer.Transform>(inner_list_pr.Child(1))); - auto func_expr = make_uniq(Identifier(std::move(exponent)), std::move(exponent_children)); + exponent_children.push_back(std::move(exponent_expr.expression)); + auto func_expr = + make_uniq(Identifier(std::move(exponent_expr.op)), std::move(exponent_children)); func_expr->IsOperatorMutable() = true; expr = std::move(func_expr); } return expr; } -string PEGTransformerFactory::TransformExponentOperator(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - return choice_pr.Cast().keyword; +BinaryExpressionTail +PEGTransformerFactory::TransformBitwiseExpressionTail(PEGTransformer &transformer, const string &bit_operator, + unique_ptr additive_expression) { + return {bit_operator, std::move(additive_expression)}; } -// CollateExpression <- AtTimeZoneExpression (CollateOperator AtTimeZoneExpression)* -unique_ptr PEGTransformerFactory::TransformCollateExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &collate_opt = list_pr.Child(1); - if (!collate_opt.HasResult()) { +BinaryExpressionTail +PEGTransformerFactory::TransformAdditiveExpressionTail(PEGTransformer &transformer, const string &term, + unique_ptr multiplicative_expression, + optional_idx query_location) { + return {term, std::move(multiplicative_expression), query_location}; +} + +BinaryExpressionTail +PEGTransformerFactory::TransformMultiplicativeExpressionTail(PEGTransformer &transformer, const string &factor, + unique_ptr exponentiation_expression) { + return {factor, std::move(exponentiation_expression)}; +} + +BinaryExpressionTail PEGTransformerFactory::TransformExponentiationExpressionTail( + PEGTransformer &transformer, const string &exponent_operator, unique_ptr collate_expression) { + return {exponent_operator, std::move(collate_expression)}; +} + +unique_ptr PEGTransformerFactory::TransformCollateExpression( + PEGTransformer &transformer, unique_ptr at_time_zone_expression, + optional>> collate_expression_tail) { + auto expr = std::move(at_time_zone_expression); + if (!collate_expression_tail) { return expr; } - auto &collate_expr_repeat = collate_opt.GetResult().Cast(); - for (auto &collate_expr_pr : collate_expr_repeat.GetChildren()) { - auto &inner_list_pr = collate_expr_pr.get().Cast(); - vector> collate_children; - auto collate_string_expr = - transformer.Transform>(inner_list_pr.Child(1)); + for (auto &collate_string_expr : *collate_expression_tail) { string collate_string; if (collate_string_expr->GetExpressionClass() == ExpressionClass::CONSTANT) { auto &const_expr = collate_string_expr->Cast(); @@ -1284,21 +1243,16 @@ unique_ptr PEGTransformerFactory::TransformCollateExpression(P return expr; } -// AtTimeZoneExpression <- PrefixExpression (AtTimeZoneOperator PrefixExpression)* -unique_ptr PEGTransformerFactory::TransformAtTimeZoneExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto &at_time_zone_opt = list_pr.Child(1); - if (!at_time_zone_opt.HasResult()) { +unique_ptr PEGTransformerFactory::TransformAtTimeZoneExpression( + PEGTransformer &transformer, unique_ptr prefix_expression, + optional>> at_time_zone_expression_tail) { + auto expr = std::move(prefix_expression); + if (!at_time_zone_expression_tail) { return expr; } - auto &at_time_zone_repeat = at_time_zone_opt.GetResult().Cast(); - for (auto &time_zone_expr : at_time_zone_repeat.GetChildren()) { - auto &inner_list_pr = time_zone_expr.get().Cast(); + for (auto &time_zone_expr : *at_time_zone_expression_tail) { vector> time_zone_children; - time_zone_children.push_back( - transformer.Transform>(inner_list_pr.Child(1))); + time_zone_children.push_back(std::move(time_zone_expr)); time_zone_children.push_back(std::move(expr)); auto func_expr = make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "timezone", std::move(time_zone_children)); @@ -1390,24 +1344,7 @@ unique_ptr PEGTransformerFactory::TransformPrefixExpression(PE } return expr; } -string PEGTransformerFactory::TransformPrefixOperator(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0); - if (StringUtil::CIEquals(choice_pr.GetResult().name, "QualifiedOperator")) { - return transformer.Transform(choice_pr.GetResult()); - } - return transformer.TransformEnum(choice_pr.GetResult()); -} - -unique_ptr PEGTransformerFactory::TransformParameter(PEGTransformer &transformer, - ParseResult &parse_result) { - // Parameter <- AnonymousParameter / NumberedParameter / ColLabelParameter - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(0).GetResult()); -} - -unique_ptr PEGTransformerFactory::TransformAnonymousParameter(PEGTransformer &transformer, - ParseResult &parse_result) { +unique_ptr PEGTransformerFactory::TransformAnonymousParameter(PEGTransformer &transformer) { // AnonymousParameter <- '?' auto expr = make_uniq(); @@ -1423,13 +1360,11 @@ unique_ptr PEGTransformerFactory::TransformAnonymousParameter( return std::move(expr); } -unique_ptr PEGTransformerFactory::TransformQuestionMarkNumberedParameter(PEGTransformer &transformer, - ParseResult &parse_result) { +unique_ptr +PEGTransformerFactory::TransformQuestionMarkNumberedParameter(PEGTransformer &transformer, + unique_ptr number_literal) { // QuestionMarkNumberedParameter <- '?' NumberLiteral - auto &list_pr = parse_result.Cast(); - auto number = transformer.Transform>(list_pr.GetChild(1)); - - auto &const_expr = number->Cast(); + auto &const_expr = number_literal->Cast(); int32_t param_number = const_expr.GetValue().GetValue(); if (param_number <= 0) { @@ -1452,13 +1387,11 @@ unique_ptr PEGTransformerFactory::TransformQuestionMarkNumbere return std::move(expr); } -unique_ptr PEGTransformerFactory::TransformNumberedParameter(PEGTransformer &transformer, - ParseResult &parse_result) { +unique_ptr +PEGTransformerFactory::TransformNumberedParameter(PEGTransformer &transformer, + unique_ptr number_literal) { // NumberedParameter <- '$' NumberLiteral - auto &list_pr = parse_result.Cast(); - auto number = transformer.Transform>(list_pr.GetChild(1)); - - auto &const_expr = number->Cast(); + auto &const_expr = number_literal->Cast(); int32_t param_number = const_expr.GetValue().GetValue(); if (param_number <= 0) { @@ -1482,10 +1415,9 @@ unique_ptr PEGTransformerFactory::TransformNumberedParameter(P } unique_ptr PEGTransformerFactory::TransformColLabelParameter(PEGTransformer &transformer, - ParseResult &parse_result) { + const string &col_label) { // ColLabelParameter <- '$' ColLabel - auto &list_pr = parse_result.Cast(); - string identifier = transformer.Transform(list_pr.GetChild(1)); + const string &identifier = col_label; auto expr = make_uniq(); idx_t known_param_index = DConstants::INVALID_INDEX; @@ -1503,11 +1435,10 @@ unique_ptr PEGTransformerFactory::TransformColLabelParameter(P return std::move(expr); } -unique_ptr PEGTransformerFactory::TransformPositionalExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto number = transformer.Transform>(list_pr.GetChild(1)); - auto &const_expr = number->Cast(); +unique_ptr +PEGTransformerFactory::TransformPositionalExpression(PEGTransformer &transformer, + unique_ptr number_literal) { + auto &const_expr = number_literal->Cast(); int32_t index = const_expr.GetValue().GetValue(); if (index <= 0) { throw ParserException("Positional reference node needs to be >= 1"); @@ -1517,154 +1448,121 @@ unique_ptr PEGTransformerFactory::TransformPositionalExpressio // LiteralExpression <- StringLiteral / NumberLiteral / 'NULL' / 'TRUE' / 'FALSE' unique_ptr PEGTransformerFactory::TransformLiteralExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &matched_rule_result = list_pr.Child(0); - if (matched_rule_result.name == "StringLiteral") { - auto &string_literal = matched_rule_result.GetResult().Cast(); + ParseResult &choice_result) { + if (choice_result.name == "StringLiteral") { + auto &string_literal = choice_result.Cast(); return string_literal.ToExpression(); } - return transformer.Transform>(matched_rule_result.GetResult()); + return transformer.Transform>(choice_result); } -// ParensExpression <- Parens(Expression) unique_ptr PEGTransformerFactory::TransformParensExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(0)); - return transformer.Transform>(extract_parens); + unique_ptr expression) { + return expression; } unique_ptr PEGTransformerFactory::TransformConstantLiteral(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto val = transformer.Transform(list_pr.Child(0).GetResult()); - return make_uniq(val); + const Value &child) { + return make_uniq(child); } -Value PEGTransformerFactory::TransformFalseLiteral(PEGTransformer &transformer, ParseResult &parse_result) { +Value PEGTransformerFactory::TransformFalseLiteral(PEGTransformer &transformer) { return Value(false); } -Value PEGTransformerFactory::TransformTrueLiteral(PEGTransformer &transformer, ParseResult &parse_result) { +Value PEGTransformerFactory::TransformTrueLiteral(PEGTransformer &transformer) { return Value(true); } -Value PEGTransformerFactory::TransformNullLiteral(PEGTransformer &transformer, ParseResult &parse_result) { +Value PEGTransformerFactory::TransformNullLiteral(PEGTransformer &transformer) { return Value(); } -Value PEGTransformerFactory::TransformUnknownLiteral(PEGTransformer &transformer, ParseResult &parse_result) { +Value PEGTransformerFactory::TransformUnknownLiteral(PEGTransformer &transformer) { return Value(); } -// SingleExpression <- LiteralExpression / -// Parameter / -// SubqueryExpression / -// SpecialFunctionExpression / -// ParenthesisExpression / -// IntervalLiteral / -// TypeLiteral / -// CaseExpression / -// StarExpression / -// CastExpression / -// GroupingExpression / -// MapExpression / -// FunctionExpression / -// ColumnReference / -// PrefixExpression / -// ListComprehensionExpression / -// ListExpression / -// StructExpression / -// PositionalExpression / -// DefaultExpression -unique_ptr PEGTransformerFactory::TransformSingleExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(0).GetResult()); -} - -ExpressionType PEGTransformerFactory::TransformLambdaOperator(PEGTransformer &transformer, ParseResult &parse_result) { - return ExpressionType::LAMBDA; -} - -unique_ptr PEGTransformerFactory::TransformIndirection(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(0).GetResult()); -} - -unique_ptr PEGTransformerFactory::TransformPostfixOperator(PEGTransformer &transformer, - ParseResult &parse_result) { +unique_ptr PEGTransformerFactory::TransformPostfixOperator(PEGTransformer &transformer) { vector> func_children; return make_uniq("factorial", std::move(func_children)); } unique_ptr PEGTransformerFactory::TransformCastOperator(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto type = transformer.Transform(list_pr.Child(1)); + const LogicalType &type) { // We input a dummy constant expression but replace this later with the real expression that precedes this post-fix // castOperator return make_uniq(type, make_uniq(Value())); } -unique_ptr PEGTransformerFactory::TransformDotOperator(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &nested_list = list_pr.Child(1); - auto &choice_pr = nested_list.Child(0); - if (choice_pr.name == "ColLabel") { - return make_uniq(transformer.Transform(choice_pr.GetResult())); - } - if (choice_pr.name == "MethodExpression") { - return transformer.Transform>(choice_pr.GetResult()); - } - throw InternalException("Unexpected rule encountered in 'DotOperator'"); +unique_ptr +PEGTransformerFactory::TransformDotMethodOperator(PEGTransformer &transformer, + unique_ptr method_expression) { + return method_expression; } -unique_ptr PEGTransformerFactory::TransformMethodExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto collabel = transformer.Transform(list_pr.Child(0)); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)).Cast(); - bool distinct = false; - transformer.TransformOptional(extract_parens, 0, distinct); - auto &function_arg_opt = extract_parens.Child(1); - vector function_children; - if (function_arg_opt.HasResult()) { - auto function_argument_list = ExtractParseResultsFromList(function_arg_opt.GetResult()); - for (auto function_argument : function_argument_list) { - function_children.push_back(transformer.Transform(function_argument)); - } - } - if (function_children.size() == 1 && ExpressionIsEmptyStar(function_children[0].GetExpression())) { +unique_ptr PEGTransformerFactory::TransformDotColumnOperator(PEGTransformer &transformer, + const string &col_label) { + return make_uniq(col_label); +} + +unique_ptr +PEGTransformerFactory::TransformMethodExpression(PEGTransformer &transformer, const string &col_label, + MethodArguments method_expression_arguments) { + if (method_expression_arguments.arguments.size() == 1 && + ExpressionIsEmptyStar(method_expression_arguments.arguments[0].GetExpression())) { // COUNT(*) gets converted into COUNT() - function_children.clear(); + method_expression_arguments.arguments.clear(); } - vector order_by; - transformer.TransformOptional>(extract_parens, 2, order_by); - bool ignore_nulls = false; - bool has_ignore_nulls_result = extract_parens.Child(3).HasResult(); - transformer.TransformOptional(extract_parens, 3, ignore_nulls); - if (has_ignore_nulls_result) { + if (method_expression_arguments.has_ignore_nulls) { throw ParserException("RESPECT/IGNORE NULLS is not supported for non-window functions"); } - auto result = make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, Identifier(collabel), - std::move(function_children)); - result->DistinctMutable() = distinct; - if (!order_by.empty()) { + auto result = make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, Identifier(col_label), + std::move(method_expression_arguments.arguments)); + result->DistinctMutable() = method_expression_arguments.distinct; + if (!method_expression_arguments.order_bys.empty()) { auto order_by_modifier = make_uniq(); - order_by_modifier->orders = std::move(order_by); + order_by_modifier->orders = std::move(method_expression_arguments.order_bys); result->OrderByMutable() = std::move(order_by_modifier); } return std::move(result); } -unique_ptr PEGTransformerFactory::TransformSliceExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto slice_bound = transformer.Transform>>(list_pr.Child(1)); +MethodArguments +PEGTransformerFactory::TransformMethodExpressionArguments(PEGTransformer &transformer, + MethodArguments method_expression_argument_list) { + return method_expression_argument_list; +} + +MethodArguments PEGTransformerFactory::TransformMethodExpressionArgumentList( + PEGTransformer &transformer, const optional &distinct_or_all, + optional> method_function_arguments, optional> order_by_clause, + const optional &ignore_or_respect_nulls) { + MethodArguments result; + if (distinct_or_all) { + result.distinct = *distinct_or_all; + } + if (method_function_arguments) { + result.arguments = std::move(*method_function_arguments); + } + if (order_by_clause) { + result.order_bys = std::move(*order_by_clause); + } + if (ignore_or_respect_nulls) { + result.has_ignore_nulls = true; + result.ignore_nulls = *ignore_or_respect_nulls; + } + return result; +} + +vector +PEGTransformerFactory::TransformMethodFunctionArguments(PEGTransformer &transformer, + vector function_argument) { + return function_argument; +} + +unique_ptr +PEGTransformerFactory::TransformSliceExpression(PEGTransformer &transformer, + vector> slice_bound) { if (slice_bound.empty()) { throw ParserException("Empty subscript '[]' is not allowed"); } @@ -1674,75 +1572,59 @@ unique_ptr PEGTransformerFactory::TransformSliceExpression(PEG return make_uniq(ExpressionType::ARRAY_SLICE, std::move(slice_bound)); } -vector> PEGTransformerFactory::TransformSliceBound(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +vector> PEGTransformerFactory::TransformSliceBound( + PEGTransformer &transformer, optional> expression, + optional> end_slice_bound, optional> step_slice_bound) { vector> slice_bounds; - auto &start_slice_opt = list_pr.Child(0); - auto &end_slice_opt = list_pr.Child(1); - auto &step_slice_opt = list_pr.Child(2); - if (!end_slice_opt.HasResult() && !step_slice_opt.HasResult()) { - if (start_slice_opt.HasResult()) { - slice_bounds.push_back(transformer.Transform>(start_slice_opt.GetResult())); + if (!end_slice_bound && !step_slice_bound) { + if (expression && *expression) { + slice_bounds.push_back(std::move(*expression)); } return slice_bounds; } - auto const_list = make_uniq(Value::LIST(LogicalType::INTEGER, vector())); - if (start_slice_opt.HasResult()) { - slice_bounds.push_back(transformer.Transform>(start_slice_opt.GetResult())); + if (expression && *expression) { + slice_bounds.push_back(std::move(*expression)); } else { - slice_bounds.push_back(const_list->Copy()); + slice_bounds.push_back(make_uniq(Value::LIST(LogicalType::INTEGER, vector()))); } - if (end_slice_opt.HasResult()) { - slice_bounds.push_back(transformer.Transform>(end_slice_opt.GetResult())); + if (end_slice_bound && *end_slice_bound) { + slice_bounds.push_back(std::move(*end_slice_bound)); } else { - slice_bounds.push_back(const_list->Copy()); + slice_bounds.push_back(make_uniq(Value::LIST(LogicalType::INTEGER, vector()))); } - if (step_slice_opt.HasResult()) { - slice_bounds.push_back(transformer.Transform>(step_slice_opt.GetResult())); + if (step_slice_bound && *step_slice_bound) { + slice_bounds.push_back(std::move(*step_slice_bound)); } return slice_bounds; } -unique_ptr PEGTransformerFactory::TransformEndSliceBound(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &nested_list_opt = list_pr.Child(1); +unique_ptr +PEGTransformerFactory::TransformEndSliceBound(PEGTransformer &transformer, + optional> end_slice_value) { // If either the lower or upper bound is not specified, we use an empty constant LIST, // which we handle in the execution. - auto const_list = make_uniq(Value::LIST(LogicalType::INTEGER, vector())); - if (nested_list_opt.HasResult()) { - auto &nested_list = nested_list_opt.GetResult().Cast(); - auto &choice_pr = nested_list.Child(0); - if (choice_pr.GetResult().type == ParseResultType::KEYWORD) { - // We have hit the '-' - return std::move(const_list); - } - if (choice_pr.GetResult().type == ParseResultType::LIST) { - return transformer.Transform>(choice_pr.GetResult()); - } - throw InternalException("Unexpected parse result type encountered"); + if (end_slice_value && *end_slice_value) { + return std::move(*end_slice_value); } - // return empty list here - return std::move(const_list); + return make_uniq(Value::LIST(LogicalType::INTEGER, vector())); } -unique_ptr PEGTransformerFactory::TransformStepSliceBound(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &expression_opt = list_pr.Child(1); - if (expression_opt.HasResult()) { - return transformer.Transform>(expression_opt.GetResult()); +unique_ptr PEGTransformerFactory::TransformEndSliceMinus(PEGTransformer &transformer) { + return make_uniq(Value::LIST(LogicalType::INTEGER, vector())); +} + +unique_ptr +PEGTransformerFactory::TransformStepSliceBound(PEGTransformer &transformer, + optional> expression) { + if (expression && *expression) { + return std::move(*expression); } return make_uniq(Value::LIST(LogicalType::INTEGER, vector())); } -unique_ptr PEGTransformerFactory::TransformTableReservedColumnName(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto table = transformer.Transform(list_pr.Child(0)); - auto column = list_pr.Child(1).identifier; - return make_uniq(column, Identifier(table)); +unique_ptr PEGTransformerFactory::TransformTableReservedColumnName( + PEGTransformer &transformer, const Identifier &table_qualification, const Identifier &reserved_column_name) { + return make_uniq(reserved_column_name, table_qualification); } Identifier PEGTransformerFactory::TransformTableQualification(PEGTransformer &transformer, @@ -1750,30 +1632,26 @@ Identifier PEGTransformerFactory::TransformTableQualification(PEGTransformer &tr return table_name; } -string PEGTransformerFactory::TransformColIdDot(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform(list_pr.GetChild(0)); +string PEGTransformerFactory::TransformColIdDot(PEGTransformer &transformer, const Identifier &col_id) { + return col_id.GetIdentifierName(); } -unique_ptr PEGTransformerFactory::TransformStarExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - +unique_ptr PEGTransformerFactory::TransformStarExpression( + PEGTransformer &transformer, const optional> &star_qualifier_list, + const optional &exclude_list, + optional>> replace_list, + const optional> &rename_list) { auto result = make_uniq(); - auto &repeat_colid_opt = list_pr.Child(0); - if (repeat_colid_opt.HasResult()) { - auto &repeat_colid = repeat_colid_opt.GetResult().Cast(); - if (repeat_colid.GetChildren().size() > 1) { + if (star_qualifier_list) { + if (star_qualifier_list->size() > 1) { throw ParserException("Did not expect more than one column in front of a star expression"); } - result->RelationNameMutable() = - Identifier(transformer.Transform(repeat_colid.Child(0))); - } - transformer.TransformOptional(list_pr, 2, result->ExcludeListMutable()); - auto &replace_list_opt = list_pr.Child(3); - if (replace_list_opt.HasResult()) { - auto replace_string_map = - transformer.Transform>>(replace_list_opt.GetResult()); - for (auto &replace_entry : replace_string_map) { + result->RelationNameMutable() = Identifier((*star_qualifier_list)[0]); + } + if (exclude_list) { + result->ExcludeListMutable() = *exclude_list; + } + if (replace_list) { + for (auto &replace_entry : *replace_list) { result->ReplaceListMutable()[Identifier(replace_entry.first)] = std::move(replace_entry.second); } for (auto &replace_entry : result->ReplaceList()) { @@ -1783,10 +1661,8 @@ unique_ptr PEGTransformerFactory::TransformStarExpression(PEGT } } } - auto &rename_list_opt = list_pr.Child(4); - if (rename_list_opt.HasResult()) { - auto rename_string_map = transformer.Transform>(rename_list_opt.GetResult()); - for (auto &rename_entry : rename_string_map) { + if (rename_list) { + for (auto &rename_entry : *rename_list) { result->RenameListMutable()[rename_entry.first] = Identifier(rename_entry.second); } for (auto &rename_column : result->RenameList()) { @@ -1804,74 +1680,66 @@ unique_ptr PEGTransformerFactory::TransformStarExpression(PEGT } qualified_column_set_t PEGTransformerFactory::TransformExcludeList(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform(list_pr.Child(1).GetResult()); + const qualified_column_set_t &exclude_names) { + return exclude_names; } -qualified_column_set_t PEGTransformerFactory::TransformExcludeNameList(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(0)); - auto exclude_name_list = ExtractParseResultsFromList(extract_parens); +qualified_column_set_t +PEGTransformerFactory::TransformExcludeNameList(PEGTransformer &transformer, + const vector &exclude_name) { qualified_column_set_t result; - for (auto exclude_name : exclude_name_list) { - auto exclude_column = transformer.Transform(exclude_name); + for (auto &exclude_column : exclude_name) { if (result.find(exclude_column) != result.end()) { throw ParserException("Duplicate entry \"%s\" in EXCLUDE list", exclude_column.ToString()); } - result.insert(std::move(exclude_column)); + result.insert(exclude_column); } return result; } qualified_column_set_t PEGTransformerFactory::TransformExcludeNameSingle(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); + const QualifiedColumnName &exclude_name) { qualified_column_set_t result; - result.insert(transformer.Transform(list_pr.Child(0))); + result.insert(exclude_name); return result; } -QualifiedColumnName PEGTransformerFactory::TransformExcludeName(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - if (StringUtil::CIEquals(choice_pr.name, "dottedidentifier")) { - auto result = transformer.Transform>(choice_pr); - auto result_string = StringUtil::Join(result, "."); - return QualifiedColumnName::Parse(result_string); - } else if (StringUtil::CIEquals(choice_pr.name, "colidorstring")) { - auto result = transformer.Transform(choice_pr); - return QualifiedColumnName(Identifier(result)); - } else { - throw InternalException("Unexpected option encountered for ExcludeName"); - } +QualifiedColumnName PEGTransformerFactory::TransformExcludeDottedName(PEGTransformer &transformer, + const vector &dotted_identifier) { + auto result_string = StringUtil::Join(dotted_identifier, "."); + return QualifiedColumnName::Parse(result_string); +} + +QualifiedColumnName PEGTransformerFactory::TransformExcludeColumnName(PEGTransformer &transformer, + const Identifier &col_id_or_string) { + return QualifiedColumnName(col_id_or_string); } unique_ptr PEGTransformerFactory::TransformOverClause(PEGTransformer &transformer, ParseResult &parse_result) { + if (transformer.in_window_definition) { + throw ParserException("window functions are not allowed in window definitions"); + } auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(1)); + transformer.in_window_definition = true; + auto window_frame = transformer.Transform>(list_pr.GetChild(1)); + transformer.in_window_definition = false; + return window_frame; } unique_ptr PEGTransformerFactory::TransformWindowFrame(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0); - if (choice_pr.GetResult().type == ParseResultType::IDENTIFIER) { - auto window_name = choice_pr.GetResult().Cast().identifier; + ParseResult &choice_result) { + if (choice_result.type == ParseResultType::IDENTIFIER) { + auto window_name = choice_result.Cast().identifier; return transformer.GetWindowClause(window_name); } - return transformer.Transform>(choice_pr.GetResult()); + return transformer.Transform>(choice_result); } unique_ptr PEGTransformerFactory::TransformParensIdentifier(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.GetChild(0)); - auto window_name = extract_parens.Cast().identifier; - auto window_clause = transformer.GetWindowClause(window_name); + const Identifier &identifier) { + auto window_name = identifier.GetIdentifierName(); + auto window_clause = transformer.GetWindowClause(identifier); if (window_clause->StartExpr() || window_clause->EndExpr() || !transformer.IsWindowFrameDefault(window_clause->WindowStart(), window_clause->WindowEnd())) { throw ParserException("cannot copy window \"%s\" because it has a frame clause", window_name); @@ -1879,35 +1747,31 @@ unique_ptr PEGTransformerFactory::TransformParensIdentifier(PE return window_clause; } -unique_ptr PEGTransformerFactory::TransformWindowFrameDefinition(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(0).GetResult()); +unique_ptr +PEGTransformerFactory::TransformWindowFrameContentsParens(PEGTransformer &transformer, + unique_ptr window_frame_contents) { + return window_frame_contents; } -unique_ptr PEGTransformerFactory::TransformWindowFrameContentsParens(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(0)); - return transformer.Transform>(extract_parens); +unique_ptr +PEGTransformerFactory::TransformWindowFrameNameContentsParens(PEGTransformer &transformer, + unique_ptr window_frame_name_contents) { + return window_frame_name_contents; } -unique_ptr PEGTransformerFactory::TransformWindowFrameNameContentsParens(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(0)).Cast(); - string window_name; - transformer.TransformOptional(extract_parens, 0, window_name); +unique_ptr +PEGTransformerFactory::TransformWindowFrameNameContents(PEGTransformer &transformer, + const optional &base_window_name, + unique_ptr window_frame_contents) { + if (!base_window_name) { + return window_frame_contents; + } + auto window_name = base_window_name->GetIdentifierName(); auto lower_name = StringUtil::Lower(window_name); if (lower_name == "partition" || lower_name == "range" || lower_name == "rows" || lower_name == "groups") { throw ParserException("Invalid window name \"%s\"", window_name); } - auto window_frame_contents = - transformer.Transform>(extract_parens.Child(1)); - if (window_name.empty()) { - return window_frame_contents; - } - auto copied_window = transformer.GetWindowClause(Identifier(window_name)); + auto copied_window = transformer.GetWindowClause(*base_window_name); if (copied_window->StartExpr() || copied_window->EndExpr() || !transformer.IsWindowFrameDefault(copied_window->WindowStart(), copied_window->WindowEnd())) { throw ParserException("cannot copy window \"%s\" because it has a frame clause", window_name); @@ -1933,24 +1797,20 @@ unique_ptr PEGTransformerFactory::TransformWindowFrameNameCont return copied_window; } -string PEGTransformerFactory::TransformBaseWindowName(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return list_pr.Child(0).identifier.GetIdentifierName(); +Identifier PEGTransformerFactory::TransformBaseWindowName(PEGTransformer &transformer, const Identifier &identifier) { + return identifier; } -unique_ptr PEGTransformerFactory::TransformWindowFrameContents(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +unique_ptr PEGTransformerFactory::TransformWindowFrameContents( + PEGTransformer &transformer, optional>> window_partition, + optional> order_by_clause, optional frame_clause) { //! Create a dummy result to add modifiers to auto result = make_uniq(INVALID_CATALOG, INVALID_SCHEMA, string()); - auto &partition_opt = list_pr.Child(0); - if (partition_opt.HasResult()) { - result->PartitionsMutable() = - transformer.Transform>>(partition_opt.GetResult()); - } - auto &order_by_opt = list_pr.Child(1); - if (order_by_opt.HasResult()) { - result->OrderByMutable() = transformer.Transform>(order_by_opt.GetResult()); + if (window_partition) { + result->PartitionsMutable() = std::move(*window_partition); + } + if (order_by_clause) { + result->OrderByMutable() = std::move(*order_by_clause); for (auto &order : result->OrderByMutable()) { if (order.expression->GetExpressionType() == ExpressionType::STAR) { auto &star = order.expression->Cast(); @@ -1960,14 +1820,12 @@ unique_ptr PEGTransformerFactory::TransformWindowFrameContents } } } - auto &frame_opt = list_pr.Child(2); - if (frame_opt.HasResult()) { - auto window_frame = transformer.Transform(frame_opt.GetResult()); - result->WindowStartMutable() = window_frame.start; - result->WindowEndMutable() = window_frame.end; - result->StartExprMutable() = std::move(window_frame.start_expr); - result->EndExprMutable() = std::move(window_frame.end_expr); - result->WindowExcludeMutable() = window_frame.exclude_clause; + if (frame_clause) { + result->WindowStartMutable() = frame_clause->start; + result->WindowEndMutable() = frame_clause->end; + result->StartExprMutable() = std::move(frame_clause->start_expr); + result->EndExprMutable() = std::move(frame_clause->end_expr); + result->WindowExcludeMutable() = frame_clause->exclude_clause; } else { result->WindowStartMutable() = WindowBoundary::UNBOUNDED_PRECEDING; result->WindowEndMutable() = WindowBoundary::CURRENT_ROW_RANGE; @@ -1975,11 +1833,10 @@ unique_ptr PEGTransformerFactory::TransformWindowFrameContents return result; } -WindowFrame PEGTransformerFactory::TransformFrameClause(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +WindowFrame PEGTransformerFactory::TransformFrameClause(PEGTransformer &transformer, const string &framing, + vector frame_extent, + const optional &window_exclude_clause) { WindowFrame result; - auto framing = transformer.Transform(list_pr.Child(0)); - auto frame_extent = transformer.Transform>(list_pr.Child(1)); for (auto &frame : frame_extent) { if (StringUtil::CIEquals(framing, "rows")) { if (frame.boundary == WindowBoundary::CURRENT_ROW_RANGE) { @@ -2025,48 +1882,35 @@ WindowFrame PEGTransformerFactory::TransformFrameClause(PEGTransformer &transfor result.end_expr = std::move(frame_extent[1].expr); } } - transformer.TransformOptional(list_pr, 2, result.exclude_clause); + if (window_exclude_clause) { + result.exclude_clause = *window_exclude_clause; + } return result; } -vector PEGTransformerFactory::TransformFrameExtent(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(0).GetResult()); -} - -vector PEGTransformerFactory::TransformBetweenFrameExtent(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +vector +PEGTransformerFactory::TransformBetweenFrameExtent(PEGTransformer &transformer, WindowBoundaryExpression frame_bound, + WindowBoundaryExpression frame_bound_1) { vector result; - result.push_back(transformer.Transform(list_pr.Child(1))); - result.push_back(transformer.Transform(list_pr.Child(3))); + result.push_back(std::move(frame_bound)); + result.push_back(std::move(frame_bound_1)); return result; } -vector PEGTransformerFactory::TransformSingleFrameExtent(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +vector +PEGTransformerFactory::TransformSingleFrameExtent(PEGTransformer &transformer, WindowBoundaryExpression frame_bound) { vector result; - result.push_back(transformer.Transform(list_pr.Child(0))); + result.push_back(std::move(frame_bound)); WindowBoundaryExpression end_current_row; end_current_row.boundary = WindowBoundary::INVALID; result.push_back(std::move(end_current_row)); return result; } -WindowBoundaryExpression PEGTransformerFactory::TransformFrameBound(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform(list_pr.Child(0).GetResult()); -} - WindowBoundaryExpression PEGTransformerFactory::TransformFrameUnbounded(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - bool preceding = transformer.Transform(list_pr.Child(1)); + const bool &preceding_or_following) { WindowBoundaryExpression result; - if (preceding) { + if (preceding_or_following) { result.boundary = WindowBoundary::UNBOUNDED_PRECEDING; } else { result.boundary = WindowBoundary::UNBOUNDED_FOLLOWING; @@ -2075,12 +1919,11 @@ WindowBoundaryExpression PEGTransformerFactory::TransformFrameUnbounded(PEGTrans } WindowBoundaryExpression PEGTransformerFactory::TransformFrameExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); + unique_ptr expression, + const bool &preceding_or_following) { WindowBoundaryExpression result; - result.expr = transformer.Transform>(list_pr.Child(0)); - auto is_preceding = transformer.Transform(list_pr.Child(1)); - if (is_preceding) { + result.expr = std::move(expression); + if (preceding_or_following) { // These are placeholders and will be converted to groups/rows/range later result.boundary = WindowBoundary::EXPR_PRECEDING_RANGE; } else { @@ -2089,108 +1932,104 @@ WindowBoundaryExpression PEGTransformerFactory::TransformFrameExpression(PEGTran return result; } -WindowBoundaryExpression PEGTransformerFactory::TransformFrameCurrentRow(PEGTransformer &transformer, - ParseResult &parse_result) { +WindowBoundaryExpression PEGTransformerFactory::TransformFrameCurrentRow(PEGTransformer &transformer) { WindowBoundaryExpression result; // These are placeholders and will be converted to groups/rows/range later result.boundary = WindowBoundary::CURRENT_ROW_RANGE; return result; } -bool PEGTransformerFactory::TransformPrecedingOrFollowing(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - return StringUtil::CIEquals(choice_pr.Cast().keyword, "preceding"); +bool PEGTransformerFactory::TransformPrecedingFrame(PEGTransformer &transformer) { + return true; +} + +bool PEGTransformerFactory::TransformFollowingFrame(PEGTransformer &transformer) { + return false; } WindowExcludeMode PEGTransformerFactory::TransformWindowExcludeClause(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform(list_pr.Child(1)); + const WindowExcludeMode &window_exclude_element) { + return window_exclude_element; } -string PEGTransformerFactory::TransformFraming(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - return choice_pr.Cast().keyword; +string PEGTransformerFactory::TransformRowsFraming(PEGTransformer &transformer) { + return "ROWS"; } -WindowExcludeMode PEGTransformerFactory::TransformWindowExcludeElement(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.TransformEnum(list_pr.Child(0).GetResult()); +string PEGTransformerFactory::TransformRangeFraming(PEGTransformer &transformer) { + return "RANGE"; } -vector> PEGTransformerFactory::TransformWindowPartition(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expression_list = ExtractParseResultsFromList(list_pr.Child(2)); - vector> result; - for (auto expression : expression_list) { - result.push_back(transformer.Transform>(expression)); - } - return result; +string PEGTransformerFactory::TransformGroupsFraming(PEGTransformer &transformer) { + return "GROUPS"; } -unique_ptr PEGTransformerFactory::TransformSpecialFunctionExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(0).GetResult()); +WindowExcludeMode PEGTransformerFactory::TransformExcludeCurrentRow(PEGTransformer &transformer) { + return WindowExcludeMode::CURRENT_ROW; } -unique_ptr PEGTransformerFactory::TransformCoalesceExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +WindowExcludeMode PEGTransformerFactory::TransformExcludeGroup(PEGTransformer &transformer) { + return WindowExcludeMode::GROUP; +} + +WindowExcludeMode PEGTransformerFactory::TransformExcludeTies(PEGTransformer &transformer) { + return WindowExcludeMode::TIES; +} + +WindowExcludeMode PEGTransformerFactory::TransformExcludeNoOthers(PEGTransformer &transformer) { + return WindowExcludeMode::NO_OTHER; +} + +vector> +PEGTransformerFactory::TransformWindowPartition(PEGTransformer &transformer, + vector> expression) { + return expression; +} + +unique_ptr +PEGTransformerFactory::TransformCoalesceExpression(PEGTransformer &transformer, + vector> expression) { auto result = make_uniq(ExpressionType::OPERATOR_COALESCE); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); - auto expr_list = ExtractParseResultsFromList(extract_parens); - for (auto expr : expr_list) { - result->GetChildrenMutable().push_back(transformer.Transform>(expr)); + for (auto &expr : expression) { + result->GetChildrenMutable().push_back(std::move(expr)); } return std::move(result); } unique_ptr PEGTransformerFactory::TransformUnpackExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); + unique_ptr expression) { auto result = make_uniq(ExpressionType::OPERATOR_UNPACK); - result->GetChildrenMutable().push_back(transformer.Transform>(extract_parens)); + result->GetChildrenMutable().push_back(std::move(expression)); return std::move(result); } unique_ptr PEGTransformerFactory::TransformTryExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); + unique_ptr expression) { auto result = make_uniq(ExpressionType::OPERATOR_TRY); - result->GetChildrenMutable().push_back(transformer.Transform>(extract_parens)); + result->GetChildrenMutable().push_back(std::move(expression)); return std::move(result); } -unique_ptr PEGTransformerFactory::TransformColumnsExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - bool unpack = list_pr.Child(0).HasResult(); - +unique_ptr +PEGTransformerFactory::TransformColumnsExpression(PEGTransformer &transformer, const bool &has_result, + unique_ptr expression) { + bool unpack = has_result; auto result = make_uniq(); - auto expr = - transformer.Transform>(ExtractResultFromParens(list_pr.Child(2))); - if (expr->GetExpressionType() == ExpressionType::STAR) { - auto star_expr = unique_ptr_cast(std::move(expr)); + if (expression->GetExpressionType() == ExpressionType::STAR) { + auto star_expr = unique_ptr_cast(std::move(expression)); if (star_expr->IsColumns()) { result->ExpressionMutable() = std::move(star_expr); } else { result = std::move(star_expr); } - } else if (expr->GetExpressionType() == ExpressionType::LAMBDA) { + } else if (expression->GetExpressionType() == ExpressionType::LAMBDA) { vector> children; children.push_back(make_uniq()); - children.push_back(std::move(expr)); + children.push_back(std::move(expression)); auto list_filter = make_uniq("list_filter", std::move(children)); result->ExpressionMutable() = std::move(list_filter); } else { - result->ExpressionMutable() = std::move(expr); + result->ExpressionMutable() = std::move(expression); } result->IsColumnsMutable() = true; if (unpack) { @@ -2199,282 +2038,245 @@ unique_ptr PEGTransformerFactory::TransformColumnsExpression(P return std::move(result); } -unique_ptr PEGTransformerFactory::TransformExtractExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_expressions = ExtractResultFromParens(list_pr.Child(1)).Cast(); - vector> expr_children; - expr_children.push_back( - transformer.Transform>(extract_expressions.Child(0))); - expr_children.push_back( - transformer.Transform>(extract_expressions.Child(2))); - return make_uniq(INVALID_CATALOG, "main", "date_part", std::move(expr_children)); +unique_ptr +PEGTransformerFactory::TransformExtractExpression(PEGTransformer &transformer, + vector> extract_arguments) { + return make_uniq(INVALID_CATALOG, "main", "date_part", std::move(extract_arguments)); } -unique_ptr PEGTransformerFactory::TransformExtractArgument(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - if (choice_pr.type == ParseResultType::IDENTIFIER) { - return make_uniq(Value(choice_pr.Cast().identifier)); - } - if (choice_pr.type == ParseResultType::STRING) { - return make_uniq(Value(choice_pr.Cast().result)); - } - auto date_part = transformer.Transform(choice_pr); - return make_uniq(EnumUtil::ToString(date_part)); +vector> +PEGTransformerFactory::TransformExtractArguments(PEGTransformer &transformer, + unique_ptr extract_argument, + unique_ptr expression) { + vector> result; + result.push_back(std::move(extract_argument)); + result.push_back(std::move(expression)); + return result; } -unique_ptr PEGTransformerFactory::TransformLambdaExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +unique_ptr +PEGTransformerFactory::TransformExtractDatePartArgument(PEGTransformer &transformer, + const DatePartSpecifier &extract_date_part) { + return make_uniq(EnumUtil::ToString(extract_date_part)); +} + +unique_ptr PEGTransformerFactory::TransformExtractIdentifierArgument(PEGTransformer &transformer, + const Identifier &identifier) { + return make_uniq(Value(identifier.GetIdentifierName())); +} - auto col_id_list = ExtractParseResultsFromList(list_pr.Child(1)); +unique_ptr PEGTransformerFactory::TransformExtractStringArgument(PEGTransformer &transformer, + const string &string_literal) { + return make_uniq(Value(string_literal)); +} + +unique_ptr PEGTransformerFactory::TransformLambdaExpression( + PEGTransformer &transformer, const vector &col_id_or_string, unique_ptr expression) { vector parameters; - for (auto colid : col_id_list) { - parameters.push_back(transformer.Transform(colid)); + for (auto ¶meter : col_id_or_string) { + parameters.push_back(parameter.GetIdentifierName()); } - auto rhs_expr = transformer.Transform>(list_pr.Child(3)); - auto result = make_uniq(parameters, std::move(rhs_expr)); + auto result = make_uniq(parameters, std::move(expression)); return std::move(result); } -unique_ptr PEGTransformerFactory::TransformNullIfExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); - auto &nested_list = extract_parens.Cast(); - vector> expr_children; - expr_children.push_back(transformer.Transform>(nested_list.Child(0))); - expr_children.push_back(transformer.Transform>(nested_list.Child(2))); - return make_uniq("nullif", std::move(expr_children)); +unique_ptr +PEGTransformerFactory::TransformNullIfExpression(PEGTransformer &transformer, + vector> null_if_arguments) { + return make_uniq("nullif", std::move(null_if_arguments)); } -unique_ptr PEGTransformerFactory::TransformRowExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); - auto &expr_list_opt = extract_parens.Cast(); - if (!expr_list_opt.HasResult()) { +vector> +PEGTransformerFactory::TransformNullIfArguments(PEGTransformer &transformer, unique_ptr expression, + unique_ptr expression_1) { + vector> result; + result.push_back(std::move(expression)); + result.push_back(std::move(expression_1)); + return result; +} + +unique_ptr +PEGTransformerFactory::TransformRowExpression(PEGTransformer &transformer, + optional>> expression) { + if (!expression) { return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "row", vector>()); } - auto expr_list = ExtractParseResultsFromList(expr_list_opt.GetResult()); - vector> results; - for (auto expr : expr_list) { - results.push_back(transformer.Transform>(expr)); - } - auto func_expr = make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "row", std::move(results)); + auto func_expr = make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "row", std::move(*expression)); return std::move(func_expr); } -unique_ptr PEGTransformerFactory::TransformSubstringExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); - auto substring_arguments = transformer.Transform>>(extract_parens); +unique_ptr +PEGTransformerFactory::TransformSubstringExpression(PEGTransformer &transformer, + vector> substring_arguments) { return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "substring", std::move(substring_arguments)); } -vector> PEGTransformerFactory::TransformSubstringArguments(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>>(list_pr.Child(0).GetResult()); -} - vector> -PEGTransformerFactory::TransformSubstringExpressionList(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - vector> results; - auto expr_list = ExtractParseResultsFromList(list_pr.Child(0)); - for (const auto expr : expr_list) { - results.push_back(transformer.Transform>(expr)); - } - return results; +PEGTransformerFactory::TransformSubstringExpressionList(PEGTransformer &transformer, + vector> expression) { + return expression; } -vector> PEGTransformerFactory::TransformSubstringParameters(PEGTransformer &transformer, - ParseResult &parse_result) { - // SubstringParameters <- Expression SubstringFromFor - auto &list_pr = parse_result.Cast(); +vector> +PEGTransformerFactory::TransformSubstringParameters(PEGTransformer &transformer, + unique_ptr expression, + vector> substring_from_for) { vector> results; - results.push_back(transformer.Transform>(list_pr.GetChild(0))); - auto from_for = transformer.Transform>>(list_pr.GetChild(1)); - for (auto &arg : from_for) { + results.push_back(std::move(expression)); + for (auto &arg : substring_from_for) { results.push_back(std::move(arg)); } return results; } -vector> PEGTransformerFactory::TransformSubstringFromFor(PEGTransformer &transformer, - ParseResult &parse_result) { - // SubstringFromFor <- SubstringFromOptionalFor / SubstringFor - auto &list_pr = parse_result.Cast(); - return transformer.Transform>>(list_pr.Child(0).GetResult()); -} - vector> -PEGTransformerFactory::TransformSubstringFromOptionalFor(PEGTransformer &transformer, ParseResult &parse_result) { - // SubstringFromOptionalFor <- FromExpression ForExpression? - auto &list_pr = parse_result.Cast(); +PEGTransformerFactory::TransformSubstringFromOptionalFor(PEGTransformer &transformer, + unique_ptr from_expression, + optional> for_expression) { vector> results; - results.push_back(transformer.Transform>(list_pr.GetChild(0))); - auto &for_opt = list_pr.Child(1); - if (for_opt.HasResult()) { - results.push_back(transformer.Transform>(for_opt.GetResult())); + results.push_back(std::move(from_expression)); + if (for_expression && *for_expression) { + results.push_back(std::move(*for_expression)); } return results; } -vector> PEGTransformerFactory::TransformSubstringFor(PEGTransformer &transformer, - ParseResult &parse_result) { - // SubstringFor <- ForExpression - // SUBSTRING(str FOR len) => substring(str, 1, len) - auto &list_pr = parse_result.Cast(); +vector> +PEGTransformerFactory::TransformSubstringFor(PEGTransformer &transformer, unique_ptr for_expression) { vector> results; results.push_back(make_uniq(Value::INTEGER(1))); - results.push_back(transformer.Transform>(list_pr.GetChild(0))); + results.push_back(std::move(for_expression)); return results; } unique_ptr PEGTransformerFactory::TransformTrimExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); - auto &inner_list = extract_parens.Cast(); + TrimArguments trim_arguments) { string function_name = "trim"; - transformer.TransformOptional(inner_list, 0, function_name); - vector> trim_expressions; - auto expr_list = ExtractParseResultsFromList(inner_list.Child(2)); - for (auto expr : expr_list) { - trim_expressions.push_back(transformer.Transform>(expr)); - } - auto &trim_source_opt = inner_list.Child(1); - if (trim_source_opt.HasResult()) { - auto trim_source_expr = transformer.Transform>(trim_source_opt.GetResult()); - if (trim_source_expr) { - trim_expressions.push_back(std::move(trim_source_expr)); - } + if (trim_arguments.trim_direction) { + function_name = *trim_arguments.trim_direction; } return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, Identifier(function_name), - std::move(trim_expressions)); + std::move(trim_arguments.expressions)); } -string PEGTransformerFactory::TransformTrimDirection(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.TransformEnum(list_pr.Child(0).GetResult()); +TrimArguments PEGTransformerFactory::TransformTrimArguments(PEGTransformer &transformer, + const optional &trim_direction, + optional> trim_source, + vector> expression) { + TrimArguments result; + result.trim_direction = trim_direction; + result.expressions = std::move(expression); + if (trim_source && *trim_source) { + result.expressions.push_back(std::move(*trim_source)); + } + return result; } -unique_ptr PEGTransformerFactory::TransformTrimSource(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &expr_opt = list_pr.Child(0); - if (expr_opt.HasResult()) { - return transformer.Transform>(expr_opt.GetResult()); +unique_ptr +PEGTransformerFactory::TransformTrimSource(PEGTransformer &transformer, + optional> expression) { + if (expression) { + return std::move(*expression); } return nullptr; } -unique_ptr PEGTransformerFactory::TransformOverlayExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - // OverlayExpression <- 'OVERLAY' Parens(OverlayArguments) - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); - auto args = transformer.Transform>>(extract_parens); - return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "overlay", std::move(args)); +string PEGTransformerFactory::TransformTrimBoth(PEGTransformer &transformer) { + return "trim"; } -vector> PEGTransformerFactory::TransformOverlayArguments(PEGTransformer &transformer, - ParseResult &parse_result) { - // OverlayArguments <- OverlayParameters / OverlayExpressionList - auto &list_pr = parse_result.Cast(); - return transformer.Transform>>(list_pr.Child(0).GetResult()); +string PEGTransformerFactory::TransformTrimLeading(PEGTransformer &transformer) { + return "ltrim"; } -vector> PEGTransformerFactory::TransformOverlayParameters(PEGTransformer &transformer, - ParseResult &parse_result) { - // OverlayParameters <- Expression 'PLACING' Expression FromExpression ForExpression? - // Children: 0=expr(A), 1='PLACING', 2=expr(B), 3=FromExpression, 4=optional(ForExpression) - auto &list_pr = parse_result.Cast(); +string PEGTransformerFactory::TransformTrimTrailing(PEGTransformer &transformer) { + return "rtrim"; +} + +unique_ptr +PEGTransformerFactory::TransformOverlayExpression(PEGTransformer &transformer, + vector> overlay_arguments) { + return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "overlay", std::move(overlay_arguments)); +} + +vector> PEGTransformerFactory::TransformOverlayParameters( + PEGTransformer &transformer, unique_ptr expression, unique_ptr expression_1, + unique_ptr from_expression, optional> for_expression) { vector> results; - results.push_back(transformer.Transform>(list_pr.GetChild(0))); // A - results.push_back(transformer.Transform>(list_pr.GetChild(2))); // B (after PLACING) - results.push_back(transformer.Transform>(list_pr.GetChild(3))); // C FromExpression - auto &for_opt = list_pr.Child(4); // D ForExpression? - if (for_opt.HasResult()) { - auto &for_pr = for_opt.GetResult(); - results.push_back(transformer.Transform>(for_pr)); + results.push_back(std::move(expression)); + results.push_back(std::move(expression_1)); + results.push_back(std::move(from_expression)); + if (for_expression && *for_expression) { + results.push_back(std::move(*for_expression)); } return results; } unique_ptr PEGTransformerFactory::TransformFromExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.GetChild(1)); // after FROM + unique_ptr expression) { + return expression; } unique_ptr PEGTransformerFactory::TransformForExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.GetChild(1)); // after FOR + unique_ptr expression) { + return expression; } -vector> PEGTransformerFactory::TransformOverlayExpressionList(PEGTransformer &transformer, - ParseResult &parse_result) { - // OverlayExpressionList <- List(Expression) - auto &list_pr = parse_result.Cast(); - vector> results; - auto expr_list = ExtractParseResultsFromList(list_pr.Child(0)); - for (const auto expr : expr_list) { - results.push_back(transformer.Transform>(expr)); - } - return results; +vector> +PEGTransformerFactory::TransformOverlayExpressionList(PEGTransformer &transformer, + vector> expression) { + return expression; } -unique_ptr PEGTransformerFactory::TransformPositionExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); - auto &position_values = extract_parens.Cast(); - vector> results; - //! search_string IN string - auto first_expr = transformer.Transform>(position_values.Child(0)); - auto second_expr = transformer.Transform>(position_values.Child(2)); - results.push_back(std::move(second_expr)); - results.push_back(std::move(first_expr)); - return make_uniq("position", std::move(results)); +unique_ptr +PEGTransformerFactory::TransformPositionExpression(PEGTransformer &transformer, + vector> position_arguments) { + return make_uniq("position", std::move(position_arguments)); +} + +vector> +PEGTransformerFactory::TransformPositionArguments(PEGTransformer &transformer, + unique_ptr single_expression, + unique_ptr single_expression_1) { + vector> result; + result.push_back(std::move(single_expression_1)); + result.push_back(std::move(single_expression)); + return result; } unique_ptr PEGTransformerFactory::TransformCastExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - bool try_cast = transformer.Transform(list_pr.Child(0)); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)).Cast(); - auto expr = transformer.Transform>(extract_parens.Child(0)); - auto type = transformer.Transform(extract_parens.Child(2)); - return make_uniq(type, std::move(expr), try_cast); + const bool &cast_or_try_cast, + CastArguments cast_arguments) { + return make_uniq(cast_arguments.type, std::move(cast_arguments.expression), cast_or_try_cast); } -bool PEGTransformerFactory::TransformCastOrTryCast(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0); - return StringUtil::Lower(choice_pr.GetResult().Cast().keyword) == "try_cast"; +CastArguments PEGTransformerFactory::TransformCastArguments(PEGTransformer &transformer, + unique_ptr expression, + const LogicalType &type) { + CastArguments result; + result.expression = std::move(expression); + result.type = type; + return result; } -unique_ptr PEGTransformerFactory::TransformCaseExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +bool PEGTransformerFactory::TransformCastKeyword(PEGTransformer &transformer) { + return false; +} + +bool PEGTransformerFactory::TransformTryCastKeyword(PEGTransformer &transformer) { + return true; +} + +unique_ptr PEGTransformerFactory::TransformCaseExpression( + PEGTransformer &transformer, optional> expression, vector case_when_then, + optional> case_else) { auto result = make_uniq(); - unique_ptr opt_expr; - transformer.TransformOptional>(list_pr, 1, opt_expr); - auto cases_pr = list_pr.Child(2).GetChildren(); - for (auto &case_pr : cases_pr) { - auto case_expr = transformer.Transform(case_pr); + for (auto &case_expr : case_when_then) { CaseCheck new_case; - if (opt_expr) { - new_case.when_expr = make_uniq(ExpressionType::COMPARE_EQUAL, opt_expr->Copy(), + if (expression) { + new_case.when_expr = make_uniq(ExpressionType::COMPARE_EQUAL, (*expression)->Copy(), std::move(case_expr.when_expr)); } else { new_case.when_expr = std::move(case_expr.when_expr); @@ -2482,9 +2284,8 @@ unique_ptr PEGTransformerFactory::TransformCaseExpression(PEGT new_case.then_expr = std::move(case_expr.then_expr); result->CaseChecksMutable().push_back(std::move(new_case)); } - auto &else_expr_opt = list_pr.Child(3); - if (else_expr_opt.HasResult()) { - result->ElseMutable() = transformer.Transform>(else_expr_opt.GetResult()); + if (case_else) { + result->ElseMutable() = std::move(*case_else); } else { result->ElseMutable() = make_uniq(Value()); } @@ -2492,46 +2293,47 @@ unique_ptr PEGTransformerFactory::TransformCaseExpression(PEGT } unique_ptr PEGTransformerFactory::TransformCaseElse(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(1)); + unique_ptr expression) { + return expression; } -CaseCheck PEGTransformerFactory::TransformCaseWhenThen(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +CaseCheck PEGTransformerFactory::TransformCaseWhenThen(PEGTransformer &transformer, + unique_ptr expression, + unique_ptr expression_1) { CaseCheck result; - result.when_expr = transformer.Transform>(list_pr.Child(1)); - result.then_expr = transformer.Transform>(list_pr.Child(3)); + result.when_expr = std::move(expression); + result.then_expr = std::move(expression_1); return result; } unique_ptr PEGTransformerFactory::TransformTypeLiteral(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto colid = transformer.Transform(list_pr.Child(0)); + const Identifier &col_id, + const string &string_literal) { + auto colid = col_id.GetIdentifierName(); auto type = LogicalType(TransformStringToLogicalTypeId(colid)); if (type.id() == LogicalTypeId::LIST || type.id() == LogicalTypeId::STRUCT) { throw ParserException("Cannot convert to type %s, requires exactly one type modifier", EnumUtil::ToString(type.id())); } - auto string_literal = list_pr.Child(1).result; auto child = make_uniq(Value(string_literal)); auto unbound_type = LogicalType::UNBOUND(make_uniq(colid, vector>())); auto result = make_uniq(unbound_type, std::move(child)); return std::move(result); } -unique_ptr PEGTransformerFactory::TransformDefaultExpression(PEGTransformer &transformer, - ParseResult &parse_result) { +unique_ptr PEGTransformerFactory::TransformDefaultExpression(PEGTransformer &transformer) { return make_uniq(); } -unique_ptr PEGTransformerFactory::TransformIntervalLiteral(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +unique_ptr +PEGTransformerFactory::TransformIntervalLiteral(PEGTransformer &transformer, + unique_ptr interval_parameter, + const optional &interval) { DatePartSpecifier interval_unit = DatePartSpecifier::INVALID; - transformer.TransformOptional(list_pr, 2, interval_unit); - auto expr = transformer.Transform>(list_pr.Child(1)); + if (interval) { + interval_unit = *interval; + } + auto expr = std::move(interval_parameter); auto func_name = DateTruncSimplificationRule::DatePartToFunc(interval_unit); if (func_name.empty()) { expr = make_uniq(LogicalType::INTERVAL, std::move(expr)); @@ -2552,23 +2354,17 @@ unique_ptr PEGTransformerFactory::TransformIntervalLiteral(PEG return std::move(result); } -unique_ptr PEGTransformerFactory::TransformIntervalParameter(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0).GetResult(); - if (choice_pr.type == ParseResultType::STRING) { - return make_uniq(Value(choice_pr.Cast().result)); - } - return transformer.Transform>(choice_pr); +unique_ptr PEGTransformerFactory::TransformIntervalStringParameter(PEGTransformer &transformer, + const string &string_literal) { + return make_uniq(Value(string_literal)); } -unique_ptr PEGTransformerFactory::TransformSubqueryExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - bool is_not = list_pr.Child(0).HasResult(); - bool is_exists = list_pr.Child(1).HasResult(); - auto subquery_reference = transformer.Transform>(list_pr.Child(2)); - +unique_ptr +PEGTransformerFactory::TransformSubqueryExpression(PEGTransformer &transformer, const optional &subquery_not, + const optional &subquery_exists, + unique_ptr subquery_reference) { + bool is_not = subquery_not ? *subquery_not : false; + bool is_exists = subquery_exists ? *subquery_exists : false; auto result = make_uniq(); if (is_exists) { result->GetSubqueryTypeMutable() = SubqueryType::EXISTS; @@ -2595,25 +2391,27 @@ unique_ptr PEGTransformerFactory::TransformSubqueryExpression( return std::move(result); } -unique_ptr PEGTransformerFactory::TransformMapExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto children = transformer.Transform>>(list_pr.Child(1)); - return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "map", std::move(children)); +bool PEGTransformerFactory::TransformSubqueryNot(PEGTransformer &transformer) { + return true; } -vector> PEGTransformerFactory::TransformMapStructExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +bool PEGTransformerFactory::TransformSubqueryExists(PEGTransformer &transformer) { + return true; +} + +unique_ptr +PEGTransformerFactory::TransformMapExpression(PEGTransformer &transformer, + vector> map_struct_expression) { + return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "map", std::move(map_struct_expression)); +} + +vector> PEGTransformerFactory::TransformMapStructExpression( + PEGTransformer &transformer, optional>>> map_struct_field) { vector> keys; vector> values; - auto &map_struct_opt = list_pr.Child(1); - if (map_struct_opt.HasResult()) { - auto field_list = ExtractParseResultsFromList(map_struct_opt.GetResult()); - for (auto &field : field_list) { - // Get the pair {key, value} from the field transformer - auto key_val_pair = transformer.Transform>>(field); + if (map_struct_field) { + for (auto &key_val_pair : *map_struct_field) { keys.push_back(std::move(key_val_pair[0])); values.push_back(std::move(key_val_pair[1])); } @@ -2624,32 +2422,26 @@ vector> PEGTransformerFactory::TransformMapStructEx return result; } -vector> PEGTransformerFactory::TransformMapStructField(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +vector> +PEGTransformerFactory::TransformMapStructField(PEGTransformer &transformer, unique_ptr expression, + unique_ptr expression_1) { vector> fields; - fields.push_back(transformer.Transform>(list_pr.Child(0))); - fields.push_back(transformer.Transform>(list_pr.Child(2))); + fields.push_back(std::move(expression)); + fields.push_back(std::move(expression_1)); return fields; } -unique_ptr PEGTransformerFactory::TransformListComprehensionExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - - // 1. Extract base components - auto result_expr = transformer.Transform>(list_pr.Child(1)); - auto col_list = ExtractParseResultsFromList(list_pr.Child(3)); - auto in_expr = transformer.Transform>(list_pr.Child(5)); - auto &list_comprehension_filter = list_pr.Child(6); - +unique_ptr PEGTransformerFactory::TransformListComprehensionExpression( + PEGTransformer &transformer, unique_ptr expression, const vector &col_id_or_string, + unique_ptr expression_1, optional> list_comprehension_filter) { + auto result_expr = std::move(expression); + auto in_expr = std::move(expression_1); vector lambda_columns; - for (auto col : col_list) { - lambda_columns.push_back(transformer.Transform(col)); + for (auto &col : col_id_or_string) { + lambda_columns.push_back(col.GetIdentifierName()); } - // Basic Case: No Filter - if (!list_comprehension_filter.HasResult()) { + if (!list_comprehension_filter || !*list_comprehension_filter) { auto lambda_expression = make_uniq(lambda_columns, std::move(result_expr)); vector> apply_children; apply_children.push_back(std::move(in_expr)); @@ -2658,8 +2450,7 @@ unique_ptr PEGTransformerFactory::TransformListComprehensionEx return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "list_apply", std::move(apply_children)); } - // --- WITH FILTER: 3-Stage Transformation --- - auto filter_expr = transformer.Transform>(list_comprehension_filter.GetResult()); + auto filter_expr = std::move(*list_comprehension_filter); // STAGE 1: list_apply(in_expr, x -> struct_pack(filter := ..., result := ...)) vector struct_children; @@ -2708,132 +2499,113 @@ unique_ptr PEGTransformerFactory::TransformListComprehensionEx return make_uniq(INVALID_CATALOG, DEFAULT_SCHEMA, "list_apply", std::move(stage3_apply_args)); } -unique_ptr PEGTransformerFactory::TransformListComprehensionFilter(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>(list_pr.Child(1)); +unique_ptr +PEGTransformerFactory::TransformListComprehensionFilter(PEGTransformer &transformer, + unique_ptr expression) { + return expression; } case_insensitive_map_t> -PEGTransformerFactory::TransformReplaceList(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>>( - list_pr.Child(1)); +PEGTransformerFactory::TransformReplaceList(PEGTransformer &transformer, + case_insensitive_map_t> replace_entries) { + return replace_entries; } case_insensitive_map_t> -PEGTransformerFactory::TransformReplaceEntries(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.Transform>>( - list_pr.Child(0).GetResult()); -} - -case_insensitive_map_t> -PEGTransformerFactory::TransformReplaceEntrySingle(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto replace_entry = - transformer.Transform>>(list_pr.Child(0)); +PEGTransformerFactory::TransformReplaceEntrySingle(PEGTransformer &transformer, + pair> replace_entry) { case_insensitive_map_t> entry_map; entry_map.insert(std::move(replace_entry)); return entry_map; } case_insensitive_map_t> -PEGTransformerFactory::TransformReplaceEntryList(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(0)); - auto entry_list = ExtractParseResultsFromList(extract_parens); +PEGTransformerFactory::TransformReplaceEntryList(PEGTransformer &transformer, + vector>> replace_entry) { case_insensitive_map_t> entry_map; - for (auto entry : entry_list) { - auto replace_entry = transformer.Transform>>(entry); - if (entry_map.find(replace_entry.first) != entry_map.end()) { - throw ParserException("Duplicate entry \"%s\" in REPLACE list", replace_entry.first); + for (auto &entry : replace_entry) { + if (entry_map.find(entry.first) != entry_map.end()) { + throw ParserException("Duplicate entry \"%s\" in REPLACE list", entry.first); } - entry_map.insert(std::move(replace_entry)); + entry_map.insert(std::move(entry)); } return entry_map; } -pair> PEGTransformerFactory::TransformReplaceEntry(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto expr = transformer.Transform>(list_pr.Child(0)); - auto column_reference = transformer.Transform>(list_pr.Child(2)); +pair> +PEGTransformerFactory::TransformReplaceEntry(PEGTransformer &transformer, unique_ptr expression, + unique_ptr column_reference) { if (column_reference->GetExpressionClass() != ExpressionClass::COLUMN_REF) { throw InternalException("Expected a column reference in the replace entry"); } auto &col_ref = column_reference->Cast(); auto column_name = col_ref.GetColumnName(); - return make_pair(column_name.GetIdentifierName(), std::move(expr)); + return make_pair(column_name.GetIdentifierName(), std::move(expression)); } -ExpressionType PEGTransformerFactory::TransformIsDistinctFromOp(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - if (list_pr.Child(1).HasResult()) { +ExpressionType PEGTransformerFactory::TransformIsDistinctFromOp(PEGTransformer &transformer, const bool &has_result) { + if (has_result) { return ExpressionType::COMPARE_NOT_DISTINCT_FROM; } return ExpressionType::COMPARE_DISTINCT_FROM; } -unique_ptr PEGTransformerFactory::TransformGroupingExpression(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(1)); - auto &expr_list_opt = extract_parens.Cast(); - +unique_ptr +PEGTransformerFactory::TransformGroupingExpression(PEGTransformer &transformer, const bool &grouping_or_grouping_id, + optional>> expression) { vector> grouping_expressions; - if (expr_list_opt.HasResult()) { - auto expr_list = ExtractParseResultsFromList(expr_list_opt.GetResult()); - for (auto expr : expr_list) { - grouping_expressions.push_back(transformer.Transform>(expr)); - } + if (expression) { + grouping_expressions = std::move(*expression); } auto result = make_uniq(ExpressionType::GROUPING_FUNCTION, std::move(grouping_expressions)); return std::move(result); } -qualified_column_map_t PEGTransformerFactory::TransformRenameList(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &inner_list = list_pr.Child(1); - return transformer.Transform>(inner_list.Child(0).GetResult()); +bool PEGTransformerFactory::TransformGroupingKeyword(PEGTransformer &transformer) { + return false; } -qualified_column_map_t PEGTransformerFactory::TransformRenameEntryList(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &extract_parens = ExtractResultFromParens(list_pr.Child(0)); - auto entry_list = ExtractParseResultsFromList(extract_parens); +bool PEGTransformerFactory::TransformGroupingIdKeyword(PEGTransformer &transformer) { + return true; +} + +qualified_column_map_t +PEGTransformerFactory::TransformRenameList(PEGTransformer &transformer, + const qualified_column_map_t &rename_entries) { + return rename_entries; +} + +qualified_column_map_t +PEGTransformerFactory::TransformRenameEntryList(PEGTransformer &transformer, + const vector> &rename_entry) { qualified_column_map_t result; - for (auto entry : entry_list) { - auto rename_entry = transformer.Transform>(entry); - result[rename_entry.first] = rename_entry.second; + for (auto &entry : rename_entry) { + result[entry.first] = entry.second; } return result; } -qualified_column_map_t PEGTransformerFactory::TransformSingleRenameEntry(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); +qualified_column_map_t +PEGTransformerFactory::TransformSingleRenameEntry(PEGTransformer &transformer, + const pair &rename_entry) { qualified_column_map_t result; - auto rename_entry = transformer.Transform>(list_pr.GetChild(0)); result[rename_entry.first] = rename_entry.second; return result; } pair PEGTransformerFactory::TransformRenameEntry(PEGTransformer &transformer, - ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto column_name = transformer.Transform(list_pr.GetChild(0)); - auto alias = list_pr.Child(2).identifier; - return make_pair(column_name, alias.GetIdentifierName()); + const QualifiedColumnName &exclude_name, + const Identifier &identifier) { + return make_pair(exclude_name, identifier.GetIdentifierName()); } -bool PEGTransformerFactory::TransformIgnoreOrRespectNulls(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - return transformer.TransformEnum(list_pr.Child(0).GetResult()); +bool PEGTransformerFactory::TransformIgnoreNulls(PEGTransformer &transformer) { + return true; +} + +bool PEGTransformerFactory::TransformRespectNulls(PEGTransformer &transformer) { + return false; } } // namespace duckdb diff --git a/src/duckdb/src/parser/peg/transformer/transform_generated.cpp b/src/duckdb/src/parser/peg/transformer/transform_generated.cpp index 02b2ee294..4c030764d 100644 --- a/src/duckdb/src/parser/peg/transformer/transform_generated.cpp +++ b/src/duckdb/src/parser/peg/transformer/transform_generated.cpp @@ -1,4 +1,4 @@ -// AUTO-GENERATED by scripts/parser/gen_transformer_v2.py -- DO NOT EDIT +// AUTO-GENERATED by scripts/parser/generate_transformer.py -- DO NOT EDIT #include "duckdb/parser/peg/transformer/peg_transformer.hpp" namespace duckdb { @@ -4093,6 +4093,2740 @@ unique_ptr PEGTransformerFactory::TransformImportStatement return make_uniq>>(std::move(result)); } +unique_ptr PEGTransformerFactory::TransformColumnReferenceInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto child = transformer.Transform>(choice_pr.GetResult()); + auto result = TransformColumnReference(transformer, std::move(child)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformCatalogReservedSchemaTableColumnNameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto catalog_qualification = transformer.Transform(list_pr.GetChild(0)); + auto reserved_schema_qualification = transformer.Transform(list_pr.GetChild(1)); + auto reserved_table_qualification = transformer.Transform(list_pr.GetChild(2)); + auto reserved_column_name = list_pr.GetChild(3).Cast().identifier; + auto result = + TransformCatalogReservedSchemaTableColumnName(transformer, catalog_qualification, reserved_schema_qualification, + reserved_table_qualification, reserved_column_name); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformSchemaReservedTableColumnNameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto schema_qualification = transformer.Transform(list_pr.GetChild(0)); + auto reserved_table_qualification = transformer.Transform(list_pr.GetChild(1)); + auto reserved_column_name = list_pr.GetChild(2).Cast().identifier; + auto result = TransformSchemaReservedTableColumnName(transformer, schema_qualification, + reserved_table_qualification, reserved_column_name); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformTableReservedColumnNameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto table_qualification = transformer.Transform(list_pr.GetChild(0)); + auto reserved_column_name = list_pr.GetChild(1).Cast().identifier; + auto result = TransformTableReservedColumnName(transformer, table_qualification, reserved_column_name); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFunctionExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto function_identifier = transformer.Transform(list_pr.GetChild(0)); + auto function_expression_arguments = transformer.Transform(list_pr.GetChild(1)); + optional> within_group_clause {}; + auto &within_group_clause_opt = list_pr.GetChild(2).Cast(); + if (within_group_clause_opt.HasResult()) { + auto within_group_clause_value = + transformer.Transform>(within_group_clause_opt.GetResult()); + within_group_clause = std::move(within_group_clause_value); + } + optional> filter_clause {}; + auto &filter_clause_opt = list_pr.GetChild(3).Cast(); + if (filter_clause_opt.HasResult()) { + auto filter_clause_value = transformer.Transform>(filter_clause_opt.GetResult()); + filter_clause = std::move(filter_clause_value); + } + bool has_result {}; + auto &has_result_opt = list_pr.GetChild(4).Cast(); + has_result = has_result_opt.HasResult(); + optional> over_clause {}; + auto &over_clause_opt = list_pr.GetChild(5).Cast(); + if (over_clause_opt.HasResult()) { + auto over_clause_value = transformer.Transform>(over_clause_opt.GetResult()); + over_clause = std::move(over_clause_value); + } + auto result = TransformFunctionExpression(transformer, function_identifier, + std::move(function_expression_arguments), std::move(within_group_clause), + std::move(filter_clause), has_result, std::move(over_clause)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformFunctionExpressionArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto function_expression_argument_list = + transformer.Transform(ExtractResultFromParens(list_pr.GetChild(0))); + auto result = std::move(function_expression_argument_list); + return make_uniq>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformFunctionExpressionArgumentListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional distinct_or_all {}; + auto &distinct_or_all_opt = list_pr.GetChild(0).Cast(); + if (distinct_or_all_opt.HasResult()) { + auto distinct_or_all_value = transformer.Transform(distinct_or_all_opt.GetResult()); + distinct_or_all = distinct_or_all_value; + } + optional> function_argument_list {}; + auto &function_argument_list_opt = list_pr.GetChild(1).Cast(); + if (function_argument_list_opt.HasResult()) { + auto function_argument_list_value = + transformer.Transform>(function_argument_list_opt.GetResult()); + function_argument_list = std::move(function_argument_list_value); + } + optional> order_by_clause {}; + auto &order_by_clause_opt = list_pr.GetChild(2).Cast(); + if (order_by_clause_opt.HasResult()) { + auto order_by_clause_value = transformer.Transform>(order_by_clause_opt.GetResult()); + order_by_clause = std::move(order_by_clause_value); + } + optional ignore_or_respect_nulls {}; + auto &ignore_or_respect_nulls_opt = list_pr.GetChild(3).Cast(); + if (ignore_or_respect_nulls_opt.HasResult()) { + auto ignore_or_respect_nulls_value = transformer.Transform(ignore_or_respect_nulls_opt.GetResult()); + ignore_or_respect_nulls = ignore_or_respect_nulls_value; + } + auto result = + TransformFunctionExpressionArgumentList(transformer, distinct_or_all, std::move(function_argument_list), + std::move(order_by_clause), ignore_or_respect_nulls); + return make_uniq>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformFunctionArgumentListInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector function_argument; + auto function_argument_items = ExtractParseResultsFromList(list_pr.GetChild(0)); + for (auto &function_argument_item : function_argument_items) { + auto function_argument_value = transformer.Transform(function_argument_item.get()); + function_argument.push_back(std::move(function_argument_value)); + } + auto result = std::move(function_argument); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFunctionIdentifierInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = TransformFunctionIdentifier(transformer, choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformCatalogReservedSchemaFunctionNameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto catalog_qualification = transformer.Transform(list_pr.GetChild(0)); + optional reserved_schema_qualification {}; + auto &reserved_schema_qualification_opt = list_pr.GetChild(1).Cast(); + if (reserved_schema_qualification_opt.HasResult()) { + auto reserved_schema_qualification_value = + transformer.Transform(reserved_schema_qualification_opt.GetResult()); + reserved_schema_qualification = reserved_schema_qualification_value; + } + auto reserved_function_name = list_pr.GetChild(2).Cast().identifier; + auto result = TransformCatalogReservedSchemaFunctionName(transformer, catalog_qualification, + reserved_schema_qualification, reserved_function_name); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformSchemaReservedFunctionNameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto schema_qualification = transformer.Transform(list_pr.GetChild(0)); + auto reserved_function_name = list_pr.GetChild(1).Cast().identifier; + auto result = TransformSchemaReservedFunctionName(transformer, schema_qualification, reserved_function_name); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformDistinctOrAllInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformDistinctKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformDistinctKeyword(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformAllKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformAllKeyword(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformWithinGroupClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto order_by_clause = transformer.Transform>(ExtractResultFromParens(list_pr.GetChild(2))); + auto result = TransformWithinGroupClause(transformer, std::move(order_by_clause)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFilterClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto filter_clause_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformFilterClause(transformer, std::move(filter_clause_expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformFilterClauseExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto filter_clause_contents = + transformer.Transform>(ExtractResultFromParens(list_pr.GetChild(0))); + auto result = TransformFilterClauseExpression(transformer, std::move(filter_clause_contents)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformFilterClauseContentsInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + bool has_result {}; + auto &has_result_opt = list_pr.GetChild(0).Cast(); + has_result = has_result_opt.HasResult(); + auto expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformFilterClauseContents(transformer, has_result, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformIgnoreOrRespectNullsInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformIgnoreNullsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformIgnoreNulls(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformRespectNullsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformRespectNulls(transformer); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformParenthesisExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector> expression; + auto expression_items = ExtractParseResultsFromList(ExtractResultFromParens(list_pr.GetChild(0))); + for (auto &expression_item : expression_items) { + auto expression_value = transformer.Transform>(expression_item.get()); + expression.push_back(std::move(expression_value)); + } + auto result = TransformParenthesisExpression(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformLiteralExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = TransformLiteralExpression(transformer, choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformConstantLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto child = transformer.Transform(choice_pr.GetResult()); + auto result = TransformConstantLiteral(transformer, child); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformNullLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformNullLiteral(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformTrueLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformTrueLiteral(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformFalseLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformFalseLiteral(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformCastExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto cast_or_try_cast = transformer.Transform(list_pr.GetChild(0)); + auto cast_arguments = transformer.Transform(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformCastExpression(transformer, cast_or_try_cast, std::move(cast_arguments)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformCastArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(0)); + auto type = transformer.Transform(list_pr.GetChild(2)); + auto result = TransformCastArguments(transformer, std::move(expression), type); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformCastOrTryCastInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformCastKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformCastKeyword(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformTryCastKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformTryCastKeyword(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformColIdDotInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto col_id = transformer.Transform(list_pr.GetChild(0)); + auto result = TransformColIdDot(transformer, col_id); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformStarExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional> star_qualifier_list {}; + auto &star_qualifier_list_opt = list_pr.GetChild(0).Cast(); + if (star_qualifier_list_opt.HasResult()) { + auto star_qualifier_list_value = transformer.Transform>(star_qualifier_list_opt.GetResult()); + star_qualifier_list = star_qualifier_list_value; + } + optional exclude_list {}; + auto &exclude_list_opt = list_pr.GetChild(2).Cast(); + if (exclude_list_opt.HasResult()) { + auto exclude_list_value = transformer.Transform(exclude_list_opt.GetResult()); + exclude_list = exclude_list_value; + } + optional>> replace_list {}; + auto &replace_list_opt = list_pr.GetChild(3).Cast(); + if (replace_list_opt.HasResult()) { + auto replace_list_value = + transformer.Transform>>(replace_list_opt.GetResult()); + replace_list = std::move(replace_list_value); + } + optional> rename_list {}; + auto &rename_list_opt = list_pr.GetChild(4).Cast(); + if (rename_list_opt.HasResult()) { + auto rename_list_value = transformer.Transform>(rename_list_opt.GetResult()); + rename_list = rename_list_value; + } + auto result = + TransformStarExpression(transformer, star_qualifier_list, exclude_list, std::move(replace_list), rename_list); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformStarQualifierListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector col_id_dot; + auto &col_id_dot_repeat = list_pr.GetChild(0).Cast(); + for (auto &col_id_dot_item : col_id_dot_repeat.GetChildren()) { + auto col_id_dot_value = transformer.Transform(col_id_dot_item.get()); + col_id_dot.push_back(col_id_dot_value); + } + auto result = col_id_dot; + return make_uniq>>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto exclude_names = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformExcludeList(transformer, exclude_names); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeNamesInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeNameListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector exclude_name; + auto exclude_name_items = ExtractParseResultsFromList(ExtractResultFromParens(list_pr.GetChild(0))); + for (auto &exclude_name_item : exclude_name_items) { + auto exclude_name_value = transformer.Transform(exclude_name_item.get()); + exclude_name.push_back(exclude_name_value); + } + auto result = TransformExcludeNameList(transformer, exclude_name); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeNameSingleInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto exclude_name = transformer.Transform(list_pr.GetChild(0)); + auto result = TransformExcludeNameSingle(transformer, exclude_name); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeNameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeDottedNameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto dotted_identifier = transformer.Transform>(list_pr.GetChild(0)); + auto result = TransformExcludeDottedName(transformer, dotted_identifier); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeColumnNameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto col_id_or_string = transformer.Transform(list_pr.GetChild(0)); + auto result = TransformExcludeColumnName(transformer, col_id_or_string); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformReplaceListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto replace_entries = + transformer.Transform>>(list_pr.GetChild(1)); + auto result = TransformReplaceList(transformer, std::move(replace_entries)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformReplaceEntriesInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>>(choice_pr.GetResult()); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformReplaceEntrySingleInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto replace_entry = transformer.Transform>>(list_pr.GetChild(0)); + auto result = TransformReplaceEntrySingle(transformer, std::move(replace_entry)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformReplaceEntryListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector>> replace_entry; + auto replace_entry_items = ExtractParseResultsFromList(ExtractResultFromParens(list_pr.GetChild(0))); + for (auto &replace_entry_item : replace_entry_items) { + auto replace_entry_value = + transformer.Transform>>(replace_entry_item.get()); + replace_entry.push_back(std::move(replace_entry_value)); + } + auto result = TransformReplaceEntryList(transformer, std::move(replace_entry)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformReplaceEntryInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(0)); + auto column_reference = transformer.Transform>(list_pr.GetChild(2)); + auto result = TransformReplaceEntry(transformer, std::move(expression), std::move(column_reference)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformRenameListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto rename_entries = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformRenameList(transformer, rename_entries); + return make_uniq>>(result); +} + +unique_ptr PEGTransformerFactory::TransformRenameEntriesInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(result); +} + +unique_ptr PEGTransformerFactory::TransformRenameEntryListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector> rename_entry; + auto rename_entry_items = ExtractParseResultsFromList(ExtractResultFromParens(list_pr.GetChild(0))); + for (auto &rename_entry_item : rename_entry_items) { + auto rename_entry_value = transformer.Transform>(rename_entry_item.get()); + rename_entry.push_back(rename_entry_value); + } + auto result = TransformRenameEntryList(transformer, rename_entry); + return make_uniq>>(result); +} + +unique_ptr PEGTransformerFactory::TransformSingleRenameEntryInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto rename_entry = transformer.Transform>(list_pr.GetChild(0)); + auto result = TransformSingleRenameEntry(transformer, rename_entry); + return make_uniq>>(result); +} + +unique_ptr PEGTransformerFactory::TransformRenameEntryInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto exclude_name = transformer.Transform(list_pr.GetChild(0)); + auto identifier = list_pr.GetChild(2).Cast().identifier; + auto result = TransformRenameEntry(transformer, exclude_name, identifier); + return make_uniq>>(result); +} + +unique_ptr PEGTransformerFactory::TransformSubqueryExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional subquery_not {}; + auto &subquery_not_opt = list_pr.GetChild(0).Cast(); + if (subquery_not_opt.HasResult()) { + auto subquery_not_value = transformer.Transform(subquery_not_opt.GetResult()); + subquery_not = subquery_not_value; + } + optional subquery_exists {}; + auto &subquery_exists_opt = list_pr.GetChild(1).Cast(); + if (subquery_exists_opt.HasResult()) { + auto subquery_exists_value = transformer.Transform(subquery_exists_opt.GetResult()); + subquery_exists = subquery_exists_value; + } + auto subquery_reference = transformer.Transform>(list_pr.GetChild(2)); + auto result = + TransformSubqueryExpression(transformer, subquery_not, subquery_exists, std::move(subquery_reference)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformSubqueryNotInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformSubqueryNot(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformSubqueryExistsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformSubqueryExists(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformCaseExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional> expression {}; + auto &expression_opt = list_pr.GetChild(1).Cast(); + if (expression_opt.HasResult()) { + auto expression_value = transformer.Transform>(expression_opt.GetResult()); + expression = std::move(expression_value); + } + vector case_when_then; + auto &case_when_then_repeat = list_pr.GetChild(2).Cast(); + for (auto &case_when_then_item : case_when_then_repeat.GetChildren()) { + auto case_when_then_value = transformer.Transform(case_when_then_item.get()); + case_when_then.push_back(std::move(case_when_then_value)); + } + optional> case_else {}; + auto &case_else_opt = list_pr.GetChild(3).Cast(); + if (case_else_opt.HasResult()) { + auto case_else_value = transformer.Transform>(case_else_opt.GetResult()); + case_else = std::move(case_else_value); + } + auto result = + TransformCaseExpression(transformer, std::move(expression), std::move(case_when_then), std::move(case_else)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformCaseWhenThenInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(1)); + auto expression_1 = transformer.Transform>(list_pr.GetChild(3)); + auto result = TransformCaseWhenThen(transformer, std::move(expression), std::move(expression_1)); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformCaseElseInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformCaseElse(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformTypeLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto col_id = transformer.Transform(list_pr.GetChild(0)); + auto string_literal = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformTypeLiteral(transformer, col_id, string_literal); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIntervalLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto interval_parameter = transformer.Transform>(list_pr.GetChild(1)); + optional interval {}; + auto &interval_opt = list_pr.GetChild(2).Cast(); + if (interval_opt.HasResult()) { + auto interval_value = transformer.Transform(interval_opt.GetResult()); + interval = interval_value; + } + auto result = TransformIntervalLiteral(transformer, std::move(interval_parameter), interval); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIntervalParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformIntervalStringParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto string_literal = transformer.Transform(list_pr.GetChild(0)); + auto result = TransformIntervalStringParameter(transformer, string_literal); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFrameClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto framing = transformer.Transform(list_pr.GetChild(0)); + auto frame_extent = transformer.Transform>(list_pr.GetChild(1)); + optional window_exclude_clause {}; + auto &window_exclude_clause_opt = list_pr.GetChild(2).Cast(); + if (window_exclude_clause_opt.HasResult()) { + auto window_exclude_clause_value = + transformer.Transform(window_exclude_clause_opt.GetResult()); + window_exclude_clause = window_exclude_clause_value; + } + auto result = TransformFrameClause(transformer, framing, std::move(frame_extent), window_exclude_clause); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFramingInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformRowsFramingInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformRowsFraming(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformRangeFramingInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformRangeFraming(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformGroupsFramingInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformGroupsFraming(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformFrameExtentInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformSingleFrameExtentInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto frame_bound = transformer.Transform(list_pr.GetChild(0)); + auto result = TransformSingleFrameExtent(transformer, std::move(frame_bound)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformBetweenFrameExtentInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto frame_bound = transformer.Transform(list_pr.GetChild(1)); + auto frame_bound_1 = transformer.Transform(list_pr.GetChild(3)); + auto result = TransformBetweenFrameExtent(transformer, std::move(frame_bound), std::move(frame_bound_1)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFrameBoundInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFrameUnboundedInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto preceding_or_following = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformFrameUnbounded(transformer, preceding_or_following); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFrameExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(0)); + auto preceding_or_following = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformFrameExpression(transformer, std::move(expression), preceding_or_following); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFrameCurrentRowInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformFrameCurrentRow(transformer); + return make_uniq>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformPrecedingOrFollowingInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformPrecedingFrameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformPrecedingFrame(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformFollowingFrameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformFollowingFrame(transformer); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformWindowExcludeClauseInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto window_exclude_element = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformWindowExcludeClause(transformer, window_exclude_element); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformWindowExcludeElementInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeCurrentRowInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformExcludeCurrentRow(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeGroupInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformExcludeGroup(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeTiesInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformExcludeTies(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformExcludeNoOthersInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformExcludeNoOthers(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformWindowFrameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = TransformWindowFrame(transformer, choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformParensIdentifierInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto identifier = ExtractResultFromParens(list_pr.GetChild(0)).Cast().identifier; + auto result = TransformParensIdentifier(transformer, identifier); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformWindowFrameDefinitionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformWindowFrameNameContentsParensInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto window_frame_name_contents = + transformer.Transform>(ExtractResultFromParens(list_pr.GetChild(0))); + auto result = TransformWindowFrameNameContentsParens(transformer, std::move(window_frame_name_contents)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformWindowFrameNameContentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional base_window_name {}; + auto &base_window_name_opt = list_pr.GetChild(0).Cast(); + if (base_window_name_opt.HasResult()) { + auto base_window_name_value = transformer.Transform(base_window_name_opt.GetResult()); + base_window_name = base_window_name_value; + } + auto window_frame_contents = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformWindowFrameNameContents(transformer, base_window_name, std::move(window_frame_contents)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformWindowFrameContentsParensInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto window_frame_contents = + transformer.Transform>(ExtractResultFromParens(list_pr.GetChild(0))); + auto result = TransformWindowFrameContentsParens(transformer, std::move(window_frame_contents)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformWindowFrameContentsInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional>> window_partition {}; + auto &window_partition_opt = list_pr.GetChild(0).Cast(); + if (window_partition_opt.HasResult()) { + auto window_partition_value = + transformer.Transform>>(window_partition_opt.GetResult()); + window_partition = std::move(window_partition_value); + } + optional> order_by_clause {}; + auto &order_by_clause_opt = list_pr.GetChild(1).Cast(); + if (order_by_clause_opt.HasResult()) { + auto order_by_clause_value = transformer.Transform>(order_by_clause_opt.GetResult()); + order_by_clause = std::move(order_by_clause_value); + } + optional frame_clause {}; + auto &frame_clause_opt = list_pr.GetChild(2).Cast(); + if (frame_clause_opt.HasResult()) { + auto frame_clause_value = transformer.Transform(frame_clause_opt.GetResult()); + frame_clause = std::move(frame_clause_value); + } + auto result = TransformWindowFrameContents(transformer, std::move(window_partition), std::move(order_by_clause), + std::move(frame_clause)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformBaseWindowNameInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto identifier = list_pr.GetChild(0).Cast().identifier; + auto result = TransformBaseWindowName(transformer, identifier); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformWindowPartitionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector> expression; + auto expression_items = ExtractParseResultsFromList(list_pr.GetChild(2)); + for (auto &expression_item : expression_items) { + auto expression_value = transformer.Transform>(expression_item.get()); + expression.push_back(std::move(expression_value)); + } + auto result = TransformWindowPartition(transformer, std::move(expression)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformListExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformArrayBoundedListExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + bool has_result {}; + auto &has_result_opt = list_pr.GetChild(0).Cast(); + has_result = has_result_opt.HasResult(); + auto bounded_list_expression = transformer.Transform>>(list_pr.GetChild(1)); + auto result = TransformArrayBoundedListExpression(transformer, has_result, std::move(bounded_list_expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformArrayParensSelectInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto select_statement_internal = + transformer.Transform>(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformArrayParensSelect(transformer, std::move(select_statement_internal)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformBoundedListExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional>> expression {}; + auto &expression_opt = list_pr.GetChild(1).Cast(); + if (expression_opt.HasResult()) { + vector> expression_value; + auto expression_value_items_1 = ExtractParseResultsFromList(expression_opt.GetResult()); + for (auto &expression_value_item_1 : expression_value_items_1) { + auto expression_value_value_1 = + transformer.Transform>(expression_value_item_1.get()); + expression_value.push_back(std::move(expression_value_value_1)); + } + expression = std::move(expression_value); + } + auto result = TransformBoundedListExpression(transformer, std::move(expression)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformStructExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector struct_field; + auto struct_field_items = ExtractParseResultsFromList(list_pr.GetChild(1)); + for (auto &struct_field_item : struct_field_items) { + auto struct_field_value = transformer.Transform(struct_field_item.get()); + struct_field.push_back(std::move(struct_field_value)); + } + auto result = TransformStructExpression(transformer, std::move(struct_field)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformStructFieldInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto col_id_or_string = transformer.Transform(list_pr.GetChild(0)); + auto expression = transformer.Transform>(list_pr.GetChild(2)); + auto result = TransformStructField(transformer, col_id_or_string, std::move(expression)); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformMapExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto map_struct_expression = transformer.Transform>>(list_pr.GetChild(1)); + auto result = TransformMapExpression(transformer, std::move(map_struct_expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformMapStructExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional>>> map_struct_field {}; + auto &map_struct_field_opt = list_pr.GetChild(1).Cast(); + if (map_struct_field_opt.HasResult()) { + vector>> map_struct_field_value; + auto map_struct_field_value_items_1 = ExtractParseResultsFromList(map_struct_field_opt.GetResult()); + for (auto &map_struct_field_value_item_1 : map_struct_field_value_items_1) { + auto map_struct_field_value_value_1 = + transformer.Transform>>(map_struct_field_value_item_1.get()); + map_struct_field_value.push_back(std::move(map_struct_field_value_value_1)); + } + map_struct_field = std::move(map_struct_field_value); + } + auto result = TransformMapStructExpression(transformer, std::move(map_struct_field)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformMapStructFieldInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(0)); + auto expression_1 = transformer.Transform>(list_pr.GetChild(2)); + auto result = TransformMapStructField(transformer, std::move(expression), std::move(expression_1)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformGroupingExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto grouping_or_grouping_id = transformer.Transform(list_pr.GetChild(0)); + optional>> expression {}; + auto &expression_opt = ExtractResultFromParens(list_pr.GetChild(1)).Cast(); + if (expression_opt.HasResult()) { + vector> expression_value; + auto expression_value_items_1 = ExtractParseResultsFromList(expression_opt.GetResult()); + for (auto &expression_value_item_1 : expression_value_items_1) { + auto expression_value_value_1 = + transformer.Transform>(expression_value_item_1.get()); + expression_value.push_back(std::move(expression_value_value_1)); + } + expression = std::move(expression_value); + } + auto result = TransformGroupingExpression(transformer, grouping_or_grouping_id, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformGroupingOrGroupingIdInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformGroupingKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformGroupingKeyword(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformGroupingIdKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformGroupingIdKeyword(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformQuestionMarkNumberedParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto number_literal = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformQuestionMarkNumberedParameter(transformer, std::move(number_literal)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformAnonymousParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformAnonymousParameter(transformer); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformNumberedParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto number_literal = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformNumberedParameter(transformer, std::move(number_literal)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformColLabelParameterInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto col_label = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformColLabelParameter(transformer, col_label); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformPositionalExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto number_literal = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformPositionalExpression(transformer, std::move(number_literal)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformDefaultExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformDefaultExpression(transformer); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformListComprehensionExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(1)); + vector col_id_or_string; + auto col_id_or_string_items = ExtractParseResultsFromList(list_pr.GetChild(3)); + for (auto &col_id_or_string_item : col_id_or_string_items) { + auto col_id_or_string_value = transformer.Transform(col_id_or_string_item.get()); + col_id_or_string.push_back(col_id_or_string_value); + } + auto expression_1 = transformer.Transform>(list_pr.GetChild(5)); + optional> list_comprehension_filter {}; + auto &list_comprehension_filter_opt = list_pr.GetChild(6).Cast(); + if (list_comprehension_filter_opt.HasResult()) { + auto list_comprehension_filter_value = + transformer.Transform>(list_comprehension_filter_opt.GetResult()); + list_comprehension_filter = std::move(list_comprehension_filter_value); + } + auto result = TransformListComprehensionExpression(transformer, std::move(expression), col_id_or_string, + std::move(expression_1), std::move(list_comprehension_filter)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformListComprehensionFilterInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformListComprehensionFilter(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformParensExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(ExtractResultFromParens(list_pr.GetChild(0))); + auto result = TransformParensExpression(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformSingleExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformColumnDefaultExprInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto col_def_or_expr = transformer.Transform>(list_pr.GetChild(0)); + auto result = std::move(col_def_or_expr); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformLambdaArrowExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto logical_or_expression = transformer.Transform>(list_pr.GetChild(0)); + optional>> single_arrow_pair {}; + auto &single_arrow_pair_opt = list_pr.GetChild(1).Cast(); + if (single_arrow_pair_opt.HasResult()) { + vector> single_arrow_pair_value; + auto &single_arrow_pair_value_repeat_1 = single_arrow_pair_opt.GetResult().Cast(); + for (auto &single_arrow_pair_value_item_1 : single_arrow_pair_value_repeat_1.GetChildren()) { + auto single_arrow_pair_value_value_1 = + transformer.Transform>(single_arrow_pair_value_item_1.get()); + single_arrow_pair_value.push_back(std::move(single_arrow_pair_value_value_1)); + } + single_arrow_pair = std::move(single_arrow_pair_value); + } + auto result = + TransformLambdaArrowExpression(transformer, std::move(logical_or_expression), std::move(single_arrow_pair)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformSingleArrowPairInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto logical_or_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = std::move(logical_or_expression); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformLogicalOrExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto logical_and_expression = transformer.Transform>(list_pr.GetChild(0)); + optional>> logical_or_expression_tail {}; + auto &logical_or_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (logical_or_expression_tail_opt.HasResult()) { + vector> logical_or_expression_tail_value; + auto &logical_or_expression_tail_value_repeat_1 = + logical_or_expression_tail_opt.GetResult().Cast(); + for (auto &logical_or_expression_tail_value_item_1 : logical_or_expression_tail_value_repeat_1.GetChildren()) { + auto logical_or_expression_tail_value_value_1 = + transformer.Transform>(logical_or_expression_tail_value_item_1.get()); + logical_or_expression_tail_value.push_back(std::move(logical_or_expression_tail_value_value_1)); + } + logical_or_expression_tail = std::move(logical_or_expression_tail_value); + } + auto result = TransformLogicalOrExpression(transformer, std::move(logical_and_expression), + std::move(logical_or_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformLogicalOrExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto logical_and_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = std::move(logical_and_expression); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformColDefOrExprInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto col_def_and_expr = transformer.Transform>(list_pr.GetChild(0)); + optional>> col_def_or_expression_tail {}; + auto &col_def_or_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (col_def_or_expression_tail_opt.HasResult()) { + vector> col_def_or_expression_tail_value; + auto &col_def_or_expression_tail_value_repeat_1 = + col_def_or_expression_tail_opt.GetResult().Cast(); + for (auto &col_def_or_expression_tail_value_item_1 : col_def_or_expression_tail_value_repeat_1.GetChildren()) { + auto col_def_or_expression_tail_value_value_1 = + transformer.Transform>(col_def_or_expression_tail_value_item_1.get()); + col_def_or_expression_tail_value.push_back(std::move(col_def_or_expression_tail_value_value_1)); + } + col_def_or_expression_tail = std::move(col_def_or_expression_tail_value); + } + auto result = + TransformColDefOrExpr(transformer, std::move(col_def_and_expr), std::move(col_def_or_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformColDefOrExpressionTailInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto col_def_and_expr = transformer.Transform>(list_pr.GetChild(1)); + auto result = std::move(col_def_and_expr); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformLogicalAndExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto logical_not_expression = transformer.Transform>(list_pr.GetChild(0)); + optional>> logical_and_expression_tail {}; + auto &logical_and_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (logical_and_expression_tail_opt.HasResult()) { + vector> logical_and_expression_tail_value; + auto &logical_and_expression_tail_value_repeat_1 = + logical_and_expression_tail_opt.GetResult().Cast(); + for (auto &logical_and_expression_tail_value_item_1 : + logical_and_expression_tail_value_repeat_1.GetChildren()) { + auto logical_and_expression_tail_value_value_1 = + transformer.Transform>(logical_and_expression_tail_value_item_1.get()); + logical_and_expression_tail_value.push_back(std::move(logical_and_expression_tail_value_value_1)); + } + logical_and_expression_tail = std::move(logical_and_expression_tail_value); + } + auto result = TransformLogicalAndExpression(transformer, std::move(logical_not_expression), + std::move(logical_and_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformLogicalAndExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto logical_not_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = std::move(logical_not_expression); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformColDefAndExprInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto is_distinct_from_expression = transformer.Transform>(list_pr.GetChild(0)); + optional>> col_def_and_expression_tail {}; + auto &col_def_and_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (col_def_and_expression_tail_opt.HasResult()) { + vector> col_def_and_expression_tail_value; + auto &col_def_and_expression_tail_value_repeat_1 = + col_def_and_expression_tail_opt.GetResult().Cast(); + for (auto &col_def_and_expression_tail_value_item_1 : + col_def_and_expression_tail_value_repeat_1.GetChildren()) { + auto col_def_and_expression_tail_value_value_1 = + transformer.Transform>(col_def_and_expression_tail_value_item_1.get()); + col_def_and_expression_tail_value.push_back(std::move(col_def_and_expression_tail_value_value_1)); + } + col_def_and_expression_tail = std::move(col_def_and_expression_tail_value); + } + auto result = TransformColDefAndExpr(transformer, std::move(is_distinct_from_expression), + std::move(col_def_and_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformColDefAndExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto is_distinct_from_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = std::move(is_distinct_from_expression); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformLogicalNotExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional> not_expression {}; + auto ¬_expression_opt = list_pr.GetChild(0).Cast(); + if (not_expression_opt.HasResult()) { + auto not_expression_value = transformer.Transform>(not_expression_opt.GetResult()); + not_expression = std::move(not_expression_value); + } + auto is_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformLogicalNotExpression(transformer, std::move(not_expression), std::move(is_expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformNotExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector not_keyword; + auto ¬_keyword_repeat = list_pr.GetChild(0).Cast(); + for (auto ¬_keyword_item : not_keyword_repeat.GetChildren()) { + auto not_keyword_value = transformer.Transform(not_keyword_item.get()); + not_keyword.push_back(not_keyword_value); + } + auto result = not_keyword; + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformNotKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformNotKeyword(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformIsExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto is_distinct_from_expression = transformer.Transform>(list_pr.GetChild(0)); + optional>> is_test {}; + auto &is_test_opt = list_pr.GetChild(1).Cast(); + if (is_test_opt.HasResult()) { + vector> is_test_value; + auto &is_test_value_repeat_1 = is_test_opt.GetResult().Cast(); + for (auto &is_test_value_item_1 : is_test_value_repeat_1.GetChildren()) { + auto is_test_value_value_1 = + transformer.Transform>(is_test_value_item_1.get()); + is_test_value.push_back(std::move(is_test_value_value_1)); + } + is_test = std::move(is_test_value); + } + auto result = TransformIsExpression(transformer, std::move(is_distinct_from_expression), std::move(is_test)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIsTestInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIsLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + bool has_result {}; + auto &has_result_opt = list_pr.GetChild(1).Cast(); + has_result = has_result_opt.HasResult(); + auto is_literal_value = transformer.Transform(list_pr.GetChild(2)); + auto result = TransformIsLiteral(transformer, has_result, is_literal_value); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIsLiteralValueInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformUnknownLiteralInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformUnknownLiteral(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformNotNullInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformNotNullKeywordInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformNotNullKeyword(transformer); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformNotNullOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformNotNullOperator(transformer); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIsNullInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto is_null_operator = transformer.Transform>(list_pr.GetChild(0)); + auto result = std::move(is_null_operator); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIsNullOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformIsNullOperator(transformer); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformIsDistinctFromExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto comparison_expression = transformer.Transform>(list_pr.GetChild(0)); + optional> is_distinct_from_tail {}; + auto &is_distinct_from_tail_opt = list_pr.GetChild(1).Cast(); + if (is_distinct_from_tail_opt.HasResult()) { + vector is_distinct_from_tail_value; + auto &is_distinct_from_tail_value_repeat_1 = is_distinct_from_tail_opt.GetResult().Cast(); + for (auto &is_distinct_from_tail_value_item_1 : is_distinct_from_tail_value_repeat_1.GetChildren()) { + auto is_distinct_from_tail_value_value_1 = + transformer.Transform(is_distinct_from_tail_value_item_1.get()); + is_distinct_from_tail_value.push_back(std::move(is_distinct_from_tail_value_value_1)); + } + is_distinct_from_tail = std::move(is_distinct_from_tail_value); + } + auto result = TransformIsDistinctFromExpression(transformer, std::move(comparison_expression), + std::move(is_distinct_from_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIsDistinctFromTailInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto is_distinct_from_op = transformer.Transform(list_pr.GetChild(0)); + auto comparison_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformIsDistinctFromTail(transformer, is_distinct_from_op, std::move(comparison_expression)); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIsDistinctFromOpInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + bool has_result {}; + auto &has_result_opt = list_pr.GetChild(1).Cast(); + has_result = has_result_opt.HasResult(); + auto result = TransformIsDistinctFromOp(transformer, has_result); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformComparisonExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto between_in_like_expression = transformer.Transform>(list_pr.GetChild(0)); + optional> comparison_expression_tail {}; + auto &comparison_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (comparison_expression_tail_opt.HasResult()) { + vector comparison_expression_tail_value; + auto &comparison_expression_tail_value_repeat_1 = + comparison_expression_tail_opt.GetResult().Cast(); + for (auto &comparison_expression_tail_value_item_1 : comparison_expression_tail_value_repeat_1.GetChildren()) { + auto comparison_expression_tail_value_value_1 = + transformer.Transform(comparison_expression_tail_value_item_1.get()); + comparison_expression_tail_value.push_back(std::move(comparison_expression_tail_value_value_1)); + } + comparison_expression_tail = std::move(comparison_expression_tail_value); + } + auto result = TransformComparisonExpression(transformer, std::move(between_in_like_expression), + std::move(comparison_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformComparisonExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto comparison_operator = transformer.Transform(list_pr.GetChild(0)); + optional> not_expression {}; + auto ¬_expression_opt = list_pr.GetChild(1).Cast(); + if (not_expression_opt.HasResult()) { + auto not_expression_value = transformer.Transform>(not_expression_opt.GetResult()); + not_expression = std::move(not_expression_value); + } + auto between_in_like_expression = transformer.Transform>(list_pr.GetChild(2)); + auto result = TransformComparisonExpressionTail(transformer, comparison_operator, std::move(not_expression), + std::move(between_in_like_expression)); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformComparisonOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformOperatorEqualInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformOperatorEqual(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformOperatorNotEqualInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformOperatorNotEqual(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformOperatorLessThanInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformOperatorLessThan(transformer); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformOperatorGreaterThanInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto result = TransformOperatorGreaterThan(transformer); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformOperatorLessThanEqualsInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto result = TransformOperatorLessThanEquals(transformer); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformOperatorGreaterThanEqualsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformOperatorGreaterThanEquals(transformer); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformBetweenInLikeExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto other_operator_expression = transformer.Transform>(list_pr.GetChild(0)); + optional between_in_like_op {}; + auto &between_in_like_op_opt = list_pr.GetChild(1).Cast(); + if (between_in_like_op_opt.HasResult()) { + auto between_in_like_op_value = + transformer.Transform(between_in_like_op_opt.GetResult()); + between_in_like_op = std::move(between_in_like_op_value); + } + auto result = TransformBetweenInLikeExpression(transformer, std::move(other_operator_expression), + std::move(between_in_like_op)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformBetweenInLikeOpInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + bool has_result {}; + auto &has_result_opt = list_pr.GetChild(0).Cast(); + has_result = has_result_opt.HasResult(); + auto between_in_like_op_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformBetweenInLikeOp(transformer, has_result, std::move(between_in_like_op_expression)); + return make_uniq>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformBetweenInLikeOpExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformLikeClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto like_variations = transformer.Transform(list_pr.GetChild(0)); + auto other_operator_expression = transformer.Transform>(list_pr.GetChild(1)); + optional> escape_clause {}; + auto &escape_clause_opt = list_pr.GetChild(2).Cast(); + if (escape_clause_opt.HasResult()) { + auto escape_clause_value = transformer.Transform>(escape_clause_opt.GetResult()); + escape_clause = std::move(escape_clause_value); + } + auto result = TransformLikeClause(transformer, like_variations, std::move(other_operator_expression), + std::move(escape_clause)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformEscapeClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto comparison_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformEscapeClause(transformer, std::move(comparison_expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformLikeVariationsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformLikeTokenInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformLikeToken(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformILikeTokenInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformILikeToken(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformGlobTokenInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformGlobToken(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformSimilarToTokenInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformSimilarToToken(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformNotILikeOpInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformNotILikeOp(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformNotLikeOpInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformNotLikeOp(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformNotSimilarToOpInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformNotSimilarToOp(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformInClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto in_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformInClause(transformer, std::move(in_expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformInExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformInContainsExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto other_operator_expression = transformer.Transform>(list_pr.GetChild(0)); + auto result = TransformInContainsExpression(transformer, std::move(other_operator_expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformInExpressionListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector> expression; + auto expression_items = ExtractParseResultsFromList(ExtractResultFromParens(list_pr.GetChild(0))); + for (auto &expression_item : expression_items) { + auto expression_value = transformer.Transform>(expression_item.get()); + expression.push_back(std::move(expression_value)); + } + auto result = TransformInExpressionList(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformInSelectStatementInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto select_statement_internal = + transformer.Transform>(ExtractResultFromParens(list_pr.GetChild(0))); + auto result = TransformInSelectStatement(transformer, std::move(select_statement_internal)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformBetweenClauseInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto other_operator_expression = transformer.Transform>(list_pr.GetChild(1)); + auto other_operator_expression_1 = transformer.Transform>(list_pr.GetChild(3)); + auto result = TransformBetweenClause(transformer, std::move(other_operator_expression), + std::move(other_operator_expression_1)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformOtherOperatorExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto bitwise_expression = transformer.Transform>(list_pr.GetChild(0)); + optional> other_operator_tail {}; + auto &other_operator_tail_opt = list_pr.GetChild(1).Cast(); + if (other_operator_tail_opt.HasResult()) { + vector other_operator_tail_value; + auto &other_operator_tail_value_repeat_1 = other_operator_tail_opt.GetResult().Cast(); + for (auto &other_operator_tail_value_item_1 : other_operator_tail_value_repeat_1.GetChildren()) { + auto other_operator_tail_value_value_1 = + transformer.Transform(other_operator_tail_value_item_1.get()); + other_operator_tail_value.push_back(std::move(other_operator_tail_value_value_1)); + } + other_operator_tail = std::move(other_operator_tail_value); + } + auto result = + TransformOtherOperatorExpression(transformer, std::move(bitwise_expression), std::move(other_operator_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformOtherOperatorTailInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto other_operator = transformer.Transform(list_pr.GetChild(0)); + auto bitwise_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformOtherOperatorTail(transformer, std::move(other_operator), std::move(bitwise_expression)); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformOtherOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = TransformOtherOperator(transformer, choice_pr.GetResult()); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformAnyAllOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto any_op = transformer.Transform(list_pr.GetChild(0)); + auto any_or_all = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformAnyAllOperator(transformer, any_op, any_or_all); + return make_uniq>>(result); +} + +unique_ptr PEGTransformerFactory::TransformAnyOrAllInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformSubqueryAnyInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformSubqueryAny(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformSubqueryAllInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformSubqueryAll(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformInetOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = choice_pr.GetResult().Cast().keyword; + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformJsonOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + string result = "->>"; + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformListOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = choice_pr.GetResult().Cast().keyword; + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformStringOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = choice_pr.GetResult().Cast().keyword; + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformQualifiedOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto qualified_operator_contents = transformer.Transform(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformQualifiedOperator(transformer, qualified_operator_contents); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformQualifiedOperatorContentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional> col_id_dot {}; + auto &col_id_dot_opt = list_pr.GetChild(0).Cast(); + if (col_id_dot_opt.HasResult()) { + vector col_id_dot_value; + auto &col_id_dot_value_repeat_1 = col_id_dot_opt.GetResult().Cast(); + for (auto &col_id_dot_value_item_1 : col_id_dot_value_repeat_1.GetChildren()) { + auto col_id_dot_value_value_1 = transformer.Transform(col_id_dot_value_item_1.get()); + col_id_dot_value.push_back(col_id_dot_value_value_1); + } + col_id_dot = col_id_dot_value; + } + auto any_op = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformQualifiedOperatorContents(transformer, col_id_dot, any_op); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformAnyOpInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = choice_pr.GetResult().Cast().keyword; + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformBitwiseExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto additive_expression = transformer.Transform>(list_pr.GetChild(0)); + optional> bitwise_expression_tail {}; + auto &bitwise_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (bitwise_expression_tail_opt.HasResult()) { + vector bitwise_expression_tail_value; + auto &bitwise_expression_tail_value_repeat_1 = + bitwise_expression_tail_opt.GetResult().Cast(); + for (auto &bitwise_expression_tail_value_item_1 : bitwise_expression_tail_value_repeat_1.GetChildren()) { + auto bitwise_expression_tail_value_value_1 = + transformer.Transform(bitwise_expression_tail_value_item_1.get()); + bitwise_expression_tail_value.push_back(std::move(bitwise_expression_tail_value_value_1)); + } + bitwise_expression_tail = std::move(bitwise_expression_tail_value); + } + auto result = + TransformBitwiseExpression(transformer, std::move(additive_expression), std::move(bitwise_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformBitwiseExpressionTailInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto bit_operator = transformer.Transform(list_pr.GetChild(0)); + auto additive_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformBitwiseExpressionTail(transformer, bit_operator, std::move(additive_expression)); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformBitOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = choice_pr.GetResult().Cast().keyword; + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformAdditiveExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto multiplicative_expression = transformer.Transform>(list_pr.GetChild(0)); + optional> additive_expression_tail {}; + auto &additive_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (additive_expression_tail_opt.HasResult()) { + vector additive_expression_tail_value; + auto &additive_expression_tail_value_repeat_1 = + additive_expression_tail_opt.GetResult().Cast(); + for (auto &additive_expression_tail_value_item_1 : additive_expression_tail_value_repeat_1.GetChildren()) { + auto additive_expression_tail_value_value_1 = + transformer.Transform(additive_expression_tail_value_item_1.get()); + additive_expression_tail_value.push_back(std::move(additive_expression_tail_value_value_1)); + } + additive_expression_tail = std::move(additive_expression_tail_value); + } + auto result = TransformAdditiveExpression(transformer, std::move(multiplicative_expression), + std::move(additive_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformAdditiveExpressionTailInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto term = transformer.Transform(list_pr.GetChild(0)); + auto multiplicative_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = + TransformAdditiveExpressionTail(transformer, term, std::move(multiplicative_expression), parse_result.offset); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformTermInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = choice_pr.GetResult().Cast().keyword; + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformMultiplicativeExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto exponentiation_expression = transformer.Transform>(list_pr.GetChild(0)); + optional> multiplicative_expression_tail {}; + auto &multiplicative_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (multiplicative_expression_tail_opt.HasResult()) { + vector multiplicative_expression_tail_value; + auto &multiplicative_expression_tail_value_repeat_1 = + multiplicative_expression_tail_opt.GetResult().Cast(); + for (auto &multiplicative_expression_tail_value_item_1 : + multiplicative_expression_tail_value_repeat_1.GetChildren()) { + auto multiplicative_expression_tail_value_value_1 = + transformer.Transform(multiplicative_expression_tail_value_item_1.get()); + multiplicative_expression_tail_value.push_back(std::move(multiplicative_expression_tail_value_value_1)); + } + multiplicative_expression_tail = std::move(multiplicative_expression_tail_value); + } + auto result = TransformMultiplicativeExpression(transformer, std::move(exponentiation_expression), + std::move(multiplicative_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformMultiplicativeExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto factor = transformer.Transform(list_pr.GetChild(0)); + auto exponentiation_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformMultiplicativeExpressionTail(transformer, factor, std::move(exponentiation_expression)); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFactorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = choice_pr.GetResult().Cast().keyword; + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformExponentiationExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto collate_expression = transformer.Transform>(list_pr.GetChild(0)); + optional> exponentiation_expression_tail {}; + auto &exponentiation_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (exponentiation_expression_tail_opt.HasResult()) { + vector exponentiation_expression_tail_value; + auto &exponentiation_expression_tail_value_repeat_1 = + exponentiation_expression_tail_opt.GetResult().Cast(); + for (auto &exponentiation_expression_tail_value_item_1 : + exponentiation_expression_tail_value_repeat_1.GetChildren()) { + auto exponentiation_expression_tail_value_value_1 = + transformer.Transform(exponentiation_expression_tail_value_item_1.get()); + exponentiation_expression_tail_value.push_back(std::move(exponentiation_expression_tail_value_value_1)); + } + exponentiation_expression_tail = std::move(exponentiation_expression_tail_value); + } + auto result = TransformExponentiationExpression(transformer, std::move(collate_expression), + std::move(exponentiation_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformExponentiationExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto exponent_operator = transformer.Transform(list_pr.GetChild(0)); + auto collate_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformExponentiationExpressionTail(transformer, exponent_operator, std::move(collate_expression)); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformExponentOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = choice_pr.GetResult().Cast().keyword; + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformCollateExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto at_time_zone_expression = transformer.Transform>(list_pr.GetChild(0)); + optional>> collate_expression_tail {}; + auto &collate_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (collate_expression_tail_opt.HasResult()) { + vector> collate_expression_tail_value; + auto &collate_expression_tail_value_repeat_1 = + collate_expression_tail_opt.GetResult().Cast(); + for (auto &collate_expression_tail_value_item_1 : collate_expression_tail_value_repeat_1.GetChildren()) { + auto collate_expression_tail_value_value_1 = + transformer.Transform>(collate_expression_tail_value_item_1.get()); + collate_expression_tail_value.push_back(std::move(collate_expression_tail_value_value_1)); + } + collate_expression_tail = std::move(collate_expression_tail_value); + } + auto result = + TransformCollateExpression(transformer, std::move(at_time_zone_expression), std::move(collate_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformCollateExpressionTailInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto at_time_zone_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = std::move(at_time_zone_expression); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformAtTimeZoneExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto prefix_expression = transformer.Transform>(list_pr.GetChild(0)); + optional>> at_time_zone_expression_tail {}; + auto &at_time_zone_expression_tail_opt = list_pr.GetChild(1).Cast(); + if (at_time_zone_expression_tail_opt.HasResult()) { + vector> at_time_zone_expression_tail_value; + auto &at_time_zone_expression_tail_value_repeat_1 = + at_time_zone_expression_tail_opt.GetResult().Cast(); + for (auto &at_time_zone_expression_tail_value_item_1 : + at_time_zone_expression_tail_value_repeat_1.GetChildren()) { + auto at_time_zone_expression_tail_value_value_1 = + transformer.Transform>(at_time_zone_expression_tail_value_item_1.get()); + at_time_zone_expression_tail_value.push_back(std::move(at_time_zone_expression_tail_value_value_1)); + } + at_time_zone_expression_tail = std::move(at_time_zone_expression_tail_value); + } + auto result = TransformAtTimeZoneExpression(transformer, std::move(prefix_expression), + std::move(at_time_zone_expression_tail)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformAtTimeZoneExpressionTailInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto prefix_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = std::move(prefix_expression); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformPrefixOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformMinusPrefixOperatorInternal(PEGTransformer &transformer, ParseResult &parse_result) { + string result = "-"; + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformPlusPrefixOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + string result = "+"; + return make_uniq>(result); +} + +unique_ptr +PEGTransformerFactory::TransformTildePrefixOperatorInternal(PEGTransformer &transformer, ParseResult &parse_result) { + string result = "~"; + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformBaseExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto single_expression = transformer.Transform>(list_pr.GetChild(0)); + optional>> indirection_list {}; + auto &indirection_list_opt = list_pr.GetChild(1).Cast(); + if (indirection_list_opt.HasResult()) { + auto indirection_list_value = + transformer.Transform>>(indirection_list_opt.GetResult()); + indirection_list = std::move(indirection_list_value); + } + auto result = TransformBaseExpression(transformer, std::move(single_expression), std::move(indirection_list)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIndirectionListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector> indirection; + auto &indirection_repeat = list_pr.GetChild(0).Cast(); + for (auto &indirection_item : indirection_repeat.GetChildren()) { + auto indirection_value = transformer.Transform>(indirection_item.get()); + indirection.push_back(std::move(indirection_value)); + } + auto result = std::move(indirection); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformIndirectionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformCastOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto type = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformCastOperator(transformer, type); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformDotOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformDotMethodOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto method_expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformDotMethodOperator(transformer, std::move(method_expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformDotColumnOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto col_label = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformDotColumnOperator(transformer, col_label); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformMethodExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto col_label = transformer.Transform(list_pr.GetChild(0)); + auto method_expression_arguments = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformMethodExpression(transformer, col_label, std::move(method_expression_arguments)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformMethodExpressionArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto method_expression_argument_list = + transformer.Transform(ExtractResultFromParens(list_pr.GetChild(0))); + auto result = TransformMethodExpressionArguments(transformer, std::move(method_expression_argument_list)); + return make_uniq>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformMethodExpressionArgumentListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional distinct_or_all {}; + auto &distinct_or_all_opt = list_pr.GetChild(0).Cast(); + if (distinct_or_all_opt.HasResult()) { + auto distinct_or_all_value = transformer.Transform(distinct_or_all_opt.GetResult()); + distinct_or_all = distinct_or_all_value; + } + optional> method_function_arguments {}; + auto &method_function_arguments_opt = list_pr.GetChild(1).Cast(); + if (method_function_arguments_opt.HasResult()) { + auto method_function_arguments_value = + transformer.Transform>(method_function_arguments_opt.GetResult()); + method_function_arguments = std::move(method_function_arguments_value); + } + optional> order_by_clause {}; + auto &order_by_clause_opt = list_pr.GetChild(2).Cast(); + if (order_by_clause_opt.HasResult()) { + auto order_by_clause_value = transformer.Transform>(order_by_clause_opt.GetResult()); + order_by_clause = std::move(order_by_clause_value); + } + optional ignore_or_respect_nulls {}; + auto &ignore_or_respect_nulls_opt = list_pr.GetChild(3).Cast(); + if (ignore_or_respect_nulls_opt.HasResult()) { + auto ignore_or_respect_nulls_value = transformer.Transform(ignore_or_respect_nulls_opt.GetResult()); + ignore_or_respect_nulls = ignore_or_respect_nulls_value; + } + auto result = + TransformMethodExpressionArgumentList(transformer, distinct_or_all, std::move(method_function_arguments), + std::move(order_by_clause), ignore_or_respect_nulls); + return make_uniq>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformMethodFunctionArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector function_argument; + auto function_argument_items = ExtractParseResultsFromList(list_pr.GetChild(0)); + for (auto &function_argument_item : function_argument_items) { + auto function_argument_value = transformer.Transform(function_argument_item.get()); + function_argument.push_back(std::move(function_argument_value)); + } + auto result = TransformMethodFunctionArguments(transformer, std::move(function_argument)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformSliceExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto slice_bound = transformer.Transform>>(list_pr.GetChild(1)); + auto result = TransformSliceExpression(transformer, std::move(slice_bound)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformSliceBoundInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional> expression {}; + auto &expression_opt = list_pr.GetChild(0).Cast(); + if (expression_opt.HasResult()) { + auto expression_value = transformer.Transform>(expression_opt.GetResult()); + expression = std::move(expression_value); + } + optional> end_slice_bound {}; + auto &end_slice_bound_opt = list_pr.GetChild(1).Cast(); + if (end_slice_bound_opt.HasResult()) { + auto end_slice_bound_value = + transformer.Transform>(end_slice_bound_opt.GetResult()); + end_slice_bound = std::move(end_slice_bound_value); + } + optional> step_slice_bound {}; + auto &step_slice_bound_opt = list_pr.GetChild(2).Cast(); + if (step_slice_bound_opt.HasResult()) { + auto step_slice_bound_value = + transformer.Transform>(step_slice_bound_opt.GetResult()); + step_slice_bound = std::move(step_slice_bound_value); + } + auto result = TransformSliceBound(transformer, std::move(expression), std::move(end_slice_bound), + std::move(step_slice_bound)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformEndSliceBoundInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional> end_slice_value {}; + auto &end_slice_value_opt = list_pr.GetChild(1).Cast(); + if (end_slice_value_opt.HasResult()) { + auto end_slice_value_value = + transformer.Transform>(end_slice_value_opt.GetResult()); + end_slice_value = std::move(end_slice_value_value); + } + auto result = TransformEndSliceBound(transformer, std::move(end_slice_value)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformEndSliceValueInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformEndSliceMinusInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformEndSliceMinus(transformer); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformStepSliceBoundInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional> expression {}; + auto &expression_opt = list_pr.GetChild(1).Cast(); + if (expression_opt.HasResult()) { + auto expression_value = transformer.Transform>(expression_opt.GetResult()); + expression = std::move(expression_value); + } + auto result = TransformStepSliceBound(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformPostfixOperatorInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformPostfixOperator(transformer); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformSpecialFunctionExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformCoalesceExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector> expression; + auto expression_items = ExtractParseResultsFromList(ExtractResultFromParens(list_pr.GetChild(1))); + for (auto &expression_item : expression_items) { + auto expression_value = transformer.Transform>(expression_item.get()); + expression.push_back(std::move(expression_value)); + } + auto result = TransformCoalesceExpression(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformUnpackExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformUnpackExpression(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformTryExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformTryExpression(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformColumnsExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + bool has_result {}; + auto &has_result_opt = list_pr.GetChild(0).Cast(); + has_result = has_result_opt.HasResult(); + auto expression = transformer.Transform>(ExtractResultFromParens(list_pr.GetChild(2))); + auto result = TransformColumnsExpression(transformer, has_result, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformExtractExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto extract_arguments = + transformer.Transform>>(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformExtractExpression(transformer, std::move(extract_arguments)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformExtractArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto extract_argument = transformer.Transform>(list_pr.GetChild(0)); + auto expression = transformer.Transform>(list_pr.GetChild(2)); + auto result = TransformExtractArguments(transformer, std::move(extract_argument), std::move(expression)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformLambdaExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector col_id_or_string; + auto col_id_or_string_items = ExtractParseResultsFromList(list_pr.GetChild(1)); + for (auto &col_id_or_string_item : col_id_or_string_items) { + auto col_id_or_string_value = transformer.Transform(col_id_or_string_item.get()); + col_id_or_string.push_back(col_id_or_string_value); + } + auto expression = transformer.Transform>(list_pr.GetChild(3)); + auto result = TransformLambdaExpression(transformer, col_id_or_string, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformNullIfExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto null_if_arguments = + transformer.Transform>>(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformNullIfExpression(transformer, std::move(null_if_arguments)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformNullIfArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(0)); + auto expression_1 = transformer.Transform>(list_pr.GetChild(2)); + auto result = TransformNullIfArguments(transformer, std::move(expression), std::move(expression_1)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformPositionExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto position_arguments = + transformer.Transform>>(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformPositionExpression(transformer, std::move(position_arguments)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformPositionArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto single_expression = transformer.Transform>(list_pr.GetChild(0)); + auto single_expression_1 = transformer.Transform>(list_pr.GetChild(2)); + auto result = TransformPositionArguments(transformer, std::move(single_expression), std::move(single_expression_1)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformRowExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional>> expression {}; + auto &expression_opt = ExtractResultFromParens(list_pr.GetChild(1)).Cast(); + if (expression_opt.HasResult()) { + vector> expression_value; + auto expression_value_items_1 = ExtractParseResultsFromList(expression_opt.GetResult()); + for (auto &expression_value_item_1 : expression_value_items_1) { + auto expression_value_value_1 = + transformer.Transform>(expression_value_item_1.get()); + expression_value.push_back(std::move(expression_value_value_1)); + } + expression = std::move(expression_value); + } + auto result = TransformRowExpression(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformSubstringExpressionInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto substring_arguments = + transformer.Transform>>(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformSubstringExpression(transformer, std::move(substring_arguments)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformSubstringArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>>(choice_pr.GetResult()); + return make_uniq>>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformSubstringExpressionListInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector> expression; + auto expression_items = ExtractParseResultsFromList(list_pr.GetChild(0)); + for (auto &expression_item : expression_items) { + auto expression_value = transformer.Transform>(expression_item.get()); + expression.push_back(std::move(expression_value)); + } + auto result = TransformSubstringExpressionList(transformer, std::move(expression)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformSubstringParametersInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(0)); + auto substring_from_for = transformer.Transform>>(list_pr.GetChild(1)); + auto result = TransformSubstringParameters(transformer, std::move(expression), std::move(substring_from_for)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformSubstringFromForInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>>(choice_pr.GetResult()); + return make_uniq>>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformSubstringFromOptionalForInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto from_expression = transformer.Transform>(list_pr.GetChild(0)); + optional> for_expression {}; + auto &for_expression_opt = list_pr.GetChild(1).Cast(); + if (for_expression_opt.HasResult()) { + auto for_expression_value = transformer.Transform>(for_expression_opt.GetResult()); + for_expression = std::move(for_expression_value); + } + auto result = TransformSubstringFromOptionalFor(transformer, std::move(from_expression), std::move(for_expression)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformSubstringForInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto for_expression = transformer.Transform>(list_pr.GetChild(0)); + auto result = TransformSubstringFor(transformer, std::move(for_expression)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformTrimExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto trim_arguments = transformer.Transform(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformTrimExpression(transformer, std::move(trim_arguments)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformTrimArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional trim_direction {}; + auto &trim_direction_opt = list_pr.GetChild(0).Cast(); + if (trim_direction_opt.HasResult()) { + auto trim_direction_value = transformer.Transform(trim_direction_opt.GetResult()); + trim_direction = trim_direction_value; + } + optional> trim_source {}; + auto &trim_source_opt = list_pr.GetChild(1).Cast(); + if (trim_source_opt.HasResult()) { + auto trim_source_value = transformer.Transform>(trim_source_opt.GetResult()); + trim_source = std::move(trim_source_value); + } + vector> expression; + auto expression_items = ExtractParseResultsFromList(list_pr.GetChild(2)); + for (auto &expression_item : expression_items) { + auto expression_value = transformer.Transform>(expression_item.get()); + expression.push_back(std::move(expression_value)); + } + auto result = TransformTrimArguments(transformer, trim_direction, std::move(trim_source), std::move(expression)); + return make_uniq>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformTrimDirectionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformTrimBothInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformTrimBoth(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformTrimLeadingInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformTrimLeading(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformTrimTrailingInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto result = TransformTrimTrailing(transformer); + return make_uniq>(result); +} + +unique_ptr PEGTransformerFactory::TransformTrimSourceInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + optional> expression {}; + auto &expression_opt = list_pr.GetChild(0).Cast(); + if (expression_opt.HasResult()) { + auto expression_value = transformer.Transform>(expression_opt.GetResult()); + expression = std::move(expression_value); + } + auto result = TransformTrimSource(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformOverlayExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto overlay_arguments = + transformer.Transform>>(ExtractResultFromParens(list_pr.GetChild(1))); + auto result = TransformOverlayExpression(transformer, std::move(overlay_arguments)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformOverlayArgumentsInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>>(choice_pr.GetResult()); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformOverlayParametersInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(0)); + auto expression_1 = transformer.Transform>(list_pr.GetChild(2)); + auto from_expression = transformer.Transform>(list_pr.GetChild(3)); + optional> for_expression {}; + auto &for_expression_opt = list_pr.GetChild(4).Cast(); + if (for_expression_opt.HasResult()) { + auto for_expression_value = transformer.Transform>(for_expression_opt.GetResult()); + for_expression = std::move(for_expression_value); + } + auto result = TransformOverlayParameters(transformer, std::move(expression), std::move(expression_1), + std::move(from_expression), std::move(for_expression)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformFromExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformFromExpression(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformForExpressionInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto expression = transformer.Transform>(list_pr.GetChild(1)); + auto result = TransformForExpression(transformer, std::move(expression)); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformOverlayExpressionListInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + vector> expression; + auto expression_items = ExtractParseResultsFromList(list_pr.GetChild(0)); + for (auto &expression_item : expression_items) { + auto expression_value = transformer.Transform>(expression_item.get()); + expression.push_back(std::move(expression_value)); + } + auto result = TransformOverlayExpressionList(transformer, std::move(expression)); + return make_uniq>>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformExtractArgumentInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform>(choice_pr.GetResult()); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformExtractDatePartArgumentInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto extract_date_part = transformer.Transform(list_pr.GetChild(0)); + auto result = TransformExtractDatePartArgument(transformer, extract_date_part); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformExtractIdentifierArgumentInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto identifier = list_pr.GetChild(0).Cast().identifier; + auto result = TransformExtractIdentifierArgument(transformer, identifier); + return make_uniq>>(std::move(result)); +} + +unique_ptr +PEGTransformerFactory::TransformExtractStringArgumentInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto string_literal = transformer.Transform(list_pr.GetChild(0)); + auto result = TransformExtractStringArgument(transformer, string_literal); + return make_uniq>>(std::move(result)); +} + +unique_ptr PEGTransformerFactory::TransformExtractDatePartInternal(PEGTransformer &transformer, + ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto &choice_pr = list_pr.Child(0); + auto result = transformer.Transform(choice_pr.GetResult()); + return make_uniq>(result); +} + unique_ptr PEGTransformerFactory::TransformInsertStatementInternal(PEGTransformer &transformer, ParseResult &parse_result) { auto &list_pr = parse_result.Cast(); @@ -7521,6 +10255,269 @@ void PEGTransformerFactory::RegisterGenerated() { {"ExportStatement", &PEGTransformerFactory::TransformExportStatementInternal}, {"ExportSource", &PEGTransformerFactory::TransformExportSourceInternal}, {"ImportStatement", &PEGTransformerFactory::TransformImportStatementInternal}, + {"ColumnReference", &PEGTransformerFactory::TransformColumnReferenceInternal}, + {"CatalogReservedSchemaTableColumnName", + &PEGTransformerFactory::TransformCatalogReservedSchemaTableColumnNameInternal}, + {"SchemaReservedTableColumnName", &PEGTransformerFactory::TransformSchemaReservedTableColumnNameInternal}, + {"TableReservedColumnName", &PEGTransformerFactory::TransformTableReservedColumnNameInternal}, + {"FunctionExpression", &PEGTransformerFactory::TransformFunctionExpressionInternal}, + {"FunctionExpressionArguments", &PEGTransformerFactory::TransformFunctionExpressionArgumentsInternal}, + {"FunctionExpressionArgumentList", &PEGTransformerFactory::TransformFunctionExpressionArgumentListInternal}, + {"FunctionArgumentList", &PEGTransformerFactory::TransformFunctionArgumentListInternal}, + {"FunctionIdentifier", &PEGTransformerFactory::TransformFunctionIdentifierInternal}, + {"CatalogReservedSchemaFunctionName", + &PEGTransformerFactory::TransformCatalogReservedSchemaFunctionNameInternal}, + {"SchemaReservedFunctionName", &PEGTransformerFactory::TransformSchemaReservedFunctionNameInternal}, + {"DistinctOrAll", &PEGTransformerFactory::TransformDistinctOrAllInternal}, + {"DistinctKeyword", &PEGTransformerFactory::TransformDistinctKeywordInternal}, + {"AllKeyword", &PEGTransformerFactory::TransformAllKeywordInternal}, + {"WithinGroupClause", &PEGTransformerFactory::TransformWithinGroupClauseInternal}, + {"FilterClause", &PEGTransformerFactory::TransformFilterClauseInternal}, + {"FilterClauseExpression", &PEGTransformerFactory::TransformFilterClauseExpressionInternal}, + {"FilterClauseContents", &PEGTransformerFactory::TransformFilterClauseContentsInternal}, + {"IgnoreOrRespectNulls", &PEGTransformerFactory::TransformIgnoreOrRespectNullsInternal}, + {"IgnoreNulls", &PEGTransformerFactory::TransformIgnoreNullsInternal}, + {"RespectNulls", &PEGTransformerFactory::TransformRespectNullsInternal}, + {"ParenthesisExpression", &PEGTransformerFactory::TransformParenthesisExpressionInternal}, + {"LiteralExpression", &PEGTransformerFactory::TransformLiteralExpressionInternal}, + {"ConstantLiteral", &PEGTransformerFactory::TransformConstantLiteralInternal}, + {"NullLiteral", &PEGTransformerFactory::TransformNullLiteralInternal}, + {"TrueLiteral", &PEGTransformerFactory::TransformTrueLiteralInternal}, + {"FalseLiteral", &PEGTransformerFactory::TransformFalseLiteralInternal}, + {"CastExpression", &PEGTransformerFactory::TransformCastExpressionInternal}, + {"CastArguments", &PEGTransformerFactory::TransformCastArgumentsInternal}, + {"CastOrTryCast", &PEGTransformerFactory::TransformCastOrTryCastInternal}, + {"CastKeyword", &PEGTransformerFactory::TransformCastKeywordInternal}, + {"TryCastKeyword", &PEGTransformerFactory::TransformTryCastKeywordInternal}, + {"ColIdDot", &PEGTransformerFactory::TransformColIdDotInternal}, + {"StarExpression", &PEGTransformerFactory::TransformStarExpressionInternal}, + {"StarQualifierList", &PEGTransformerFactory::TransformStarQualifierListInternal}, + {"ExcludeList", &PEGTransformerFactory::TransformExcludeListInternal}, + {"ExcludeNames", &PEGTransformerFactory::TransformExcludeNamesInternal}, + {"ExcludeNameList", &PEGTransformerFactory::TransformExcludeNameListInternal}, + {"ExcludeNameSingle", &PEGTransformerFactory::TransformExcludeNameSingleInternal}, + {"ExcludeName", &PEGTransformerFactory::TransformExcludeNameInternal}, + {"ExcludeDottedName", &PEGTransformerFactory::TransformExcludeDottedNameInternal}, + {"ExcludeColumnName", &PEGTransformerFactory::TransformExcludeColumnNameInternal}, + {"ReplaceList", &PEGTransformerFactory::TransformReplaceListInternal}, + {"ReplaceEntries", &PEGTransformerFactory::TransformReplaceEntriesInternal}, + {"ReplaceEntrySingle", &PEGTransformerFactory::TransformReplaceEntrySingleInternal}, + {"ReplaceEntryList", &PEGTransformerFactory::TransformReplaceEntryListInternal}, + {"ReplaceEntry", &PEGTransformerFactory::TransformReplaceEntryInternal}, + {"RenameList", &PEGTransformerFactory::TransformRenameListInternal}, + {"RenameEntries", &PEGTransformerFactory::TransformRenameEntriesInternal}, + {"RenameEntryList", &PEGTransformerFactory::TransformRenameEntryListInternal}, + {"SingleRenameEntry", &PEGTransformerFactory::TransformSingleRenameEntryInternal}, + {"RenameEntry", &PEGTransformerFactory::TransformRenameEntryInternal}, + {"SubqueryExpression", &PEGTransformerFactory::TransformSubqueryExpressionInternal}, + {"SubqueryNot", &PEGTransformerFactory::TransformSubqueryNotInternal}, + {"SubqueryExists", &PEGTransformerFactory::TransformSubqueryExistsInternal}, + {"CaseExpression", &PEGTransformerFactory::TransformCaseExpressionInternal}, + {"CaseWhenThen", &PEGTransformerFactory::TransformCaseWhenThenInternal}, + {"CaseElse", &PEGTransformerFactory::TransformCaseElseInternal}, + {"TypeLiteral", &PEGTransformerFactory::TransformTypeLiteralInternal}, + {"IntervalLiteral", &PEGTransformerFactory::TransformIntervalLiteralInternal}, + {"IntervalParameter", &PEGTransformerFactory::TransformIntervalParameterInternal}, + {"IntervalStringParameter", &PEGTransformerFactory::TransformIntervalStringParameterInternal}, + {"FrameClause", &PEGTransformerFactory::TransformFrameClauseInternal}, + {"Framing", &PEGTransformerFactory::TransformFramingInternal}, + {"RowsFraming", &PEGTransformerFactory::TransformRowsFramingInternal}, + {"RangeFraming", &PEGTransformerFactory::TransformRangeFramingInternal}, + {"GroupsFraming", &PEGTransformerFactory::TransformGroupsFramingInternal}, + {"FrameExtent", &PEGTransformerFactory::TransformFrameExtentInternal}, + {"SingleFrameExtent", &PEGTransformerFactory::TransformSingleFrameExtentInternal}, + {"BetweenFrameExtent", &PEGTransformerFactory::TransformBetweenFrameExtentInternal}, + {"FrameBound", &PEGTransformerFactory::TransformFrameBoundInternal}, + {"FrameUnbounded", &PEGTransformerFactory::TransformFrameUnboundedInternal}, + {"FrameExpression", &PEGTransformerFactory::TransformFrameExpressionInternal}, + {"FrameCurrentRow", &PEGTransformerFactory::TransformFrameCurrentRowInternal}, + {"PrecedingOrFollowing", &PEGTransformerFactory::TransformPrecedingOrFollowingInternal}, + {"PrecedingFrame", &PEGTransformerFactory::TransformPrecedingFrameInternal}, + {"FollowingFrame", &PEGTransformerFactory::TransformFollowingFrameInternal}, + {"WindowExcludeClause", &PEGTransformerFactory::TransformWindowExcludeClauseInternal}, + {"WindowExcludeElement", &PEGTransformerFactory::TransformWindowExcludeElementInternal}, + {"ExcludeCurrentRow", &PEGTransformerFactory::TransformExcludeCurrentRowInternal}, + {"ExcludeGroup", &PEGTransformerFactory::TransformExcludeGroupInternal}, + {"ExcludeTies", &PEGTransformerFactory::TransformExcludeTiesInternal}, + {"ExcludeNoOthers", &PEGTransformerFactory::TransformExcludeNoOthersInternal}, + {"WindowFrame", &PEGTransformerFactory::TransformWindowFrameInternal}, + {"ParensIdentifier", &PEGTransformerFactory::TransformParensIdentifierInternal}, + {"WindowFrameDefinition", &PEGTransformerFactory::TransformWindowFrameDefinitionInternal}, + {"WindowFrameNameContentsParens", &PEGTransformerFactory::TransformWindowFrameNameContentsParensInternal}, + {"WindowFrameNameContents", &PEGTransformerFactory::TransformWindowFrameNameContentsInternal}, + {"WindowFrameContentsParens", &PEGTransformerFactory::TransformWindowFrameContentsParensInternal}, + {"WindowFrameContents", &PEGTransformerFactory::TransformWindowFrameContentsInternal}, + {"BaseWindowName", &PEGTransformerFactory::TransformBaseWindowNameInternal}, + {"WindowPartition", &PEGTransformerFactory::TransformWindowPartitionInternal}, + {"ListExpression", &PEGTransformerFactory::TransformListExpressionInternal}, + {"ArrayBoundedListExpression", &PEGTransformerFactory::TransformArrayBoundedListExpressionInternal}, + {"ArrayParensSelect", &PEGTransformerFactory::TransformArrayParensSelectInternal}, + {"BoundedListExpression", &PEGTransformerFactory::TransformBoundedListExpressionInternal}, + {"StructExpression", &PEGTransformerFactory::TransformStructExpressionInternal}, + {"StructField", &PEGTransformerFactory::TransformStructFieldInternal}, + {"MapExpression", &PEGTransformerFactory::TransformMapExpressionInternal}, + {"MapStructExpression", &PEGTransformerFactory::TransformMapStructExpressionInternal}, + {"MapStructField", &PEGTransformerFactory::TransformMapStructFieldInternal}, + {"GroupingExpression", &PEGTransformerFactory::TransformGroupingExpressionInternal}, + {"GroupingOrGroupingId", &PEGTransformerFactory::TransformGroupingOrGroupingIdInternal}, + {"GroupingKeyword", &PEGTransformerFactory::TransformGroupingKeywordInternal}, + {"GroupingIdKeyword", &PEGTransformerFactory::TransformGroupingIdKeywordInternal}, + {"Parameter", &PEGTransformerFactory::TransformParameterInternal}, + {"QuestionMarkNumberedParameter", &PEGTransformerFactory::TransformQuestionMarkNumberedParameterInternal}, + {"AnonymousParameter", &PEGTransformerFactory::TransformAnonymousParameterInternal}, + {"NumberedParameter", &PEGTransformerFactory::TransformNumberedParameterInternal}, + {"ColLabelParameter", &PEGTransformerFactory::TransformColLabelParameterInternal}, + {"PositionalExpression", &PEGTransformerFactory::TransformPositionalExpressionInternal}, + {"DefaultExpression", &PEGTransformerFactory::TransformDefaultExpressionInternal}, + {"ListComprehensionExpression", &PEGTransformerFactory::TransformListComprehensionExpressionInternal}, + {"ListComprehensionFilter", &PEGTransformerFactory::TransformListComprehensionFilterInternal}, + {"ParensExpression", &PEGTransformerFactory::TransformParensExpressionInternal}, + {"SingleExpression", &PEGTransformerFactory::TransformSingleExpressionInternal}, + {"ColumnDefaultExpr", &PEGTransformerFactory::TransformColumnDefaultExprInternal}, + {"LambdaArrowExpression", &PEGTransformerFactory::TransformLambdaArrowExpressionInternal}, + {"SingleArrowPair", &PEGTransformerFactory::TransformSingleArrowPairInternal}, + {"LogicalOrExpression", &PEGTransformerFactory::TransformLogicalOrExpressionInternal}, + {"LogicalOrExpressionTail", &PEGTransformerFactory::TransformLogicalOrExpressionTailInternal}, + {"ColDefOrExpr", &PEGTransformerFactory::TransformColDefOrExprInternal}, + {"ColDefOrExpressionTail", &PEGTransformerFactory::TransformColDefOrExpressionTailInternal}, + {"LogicalAndExpression", &PEGTransformerFactory::TransformLogicalAndExpressionInternal}, + {"LogicalAndExpressionTail", &PEGTransformerFactory::TransformLogicalAndExpressionTailInternal}, + {"ColDefAndExpr", &PEGTransformerFactory::TransformColDefAndExprInternal}, + {"ColDefAndExpressionTail", &PEGTransformerFactory::TransformColDefAndExpressionTailInternal}, + {"LogicalNotExpression", &PEGTransformerFactory::TransformLogicalNotExpressionInternal}, + {"NotExpression", &PEGTransformerFactory::TransformNotExpressionInternal}, + {"NotKeyword", &PEGTransformerFactory::TransformNotKeywordInternal}, + {"IsExpression", &PEGTransformerFactory::TransformIsExpressionInternal}, + {"IsTest", &PEGTransformerFactory::TransformIsTestInternal}, + {"IsLiteral", &PEGTransformerFactory::TransformIsLiteralInternal}, + {"IsLiteralValue", &PEGTransformerFactory::TransformIsLiteralValueInternal}, + {"UnknownLiteral", &PEGTransformerFactory::TransformUnknownLiteralInternal}, + {"NotNull", &PEGTransformerFactory::TransformNotNullInternal}, + {"NotNullKeyword", &PEGTransformerFactory::TransformNotNullKeywordInternal}, + {"NotNullOperator", &PEGTransformerFactory::TransformNotNullOperatorInternal}, + {"IsNull", &PEGTransformerFactory::TransformIsNullInternal}, + {"IsNullOperator", &PEGTransformerFactory::TransformIsNullOperatorInternal}, + {"IsDistinctFromExpression", &PEGTransformerFactory::TransformIsDistinctFromExpressionInternal}, + {"IsDistinctFromTail", &PEGTransformerFactory::TransformIsDistinctFromTailInternal}, + {"IsDistinctFromOp", &PEGTransformerFactory::TransformIsDistinctFromOpInternal}, + {"ComparisonExpression", &PEGTransformerFactory::TransformComparisonExpressionInternal}, + {"ComparisonExpressionTail", &PEGTransformerFactory::TransformComparisonExpressionTailInternal}, + {"ComparisonOperator", &PEGTransformerFactory::TransformComparisonOperatorInternal}, + {"OperatorEqual", &PEGTransformerFactory::TransformOperatorEqualInternal}, + {"OperatorNotEqual", &PEGTransformerFactory::TransformOperatorNotEqualInternal}, + {"OperatorLessThan", &PEGTransformerFactory::TransformOperatorLessThanInternal}, + {"OperatorGreaterThan", &PEGTransformerFactory::TransformOperatorGreaterThanInternal}, + {"OperatorLessThanEquals", &PEGTransformerFactory::TransformOperatorLessThanEqualsInternal}, + {"OperatorGreaterThanEquals", &PEGTransformerFactory::TransformOperatorGreaterThanEqualsInternal}, + {"BetweenInLikeExpression", &PEGTransformerFactory::TransformBetweenInLikeExpressionInternal}, + {"BetweenInLikeOp", &PEGTransformerFactory::TransformBetweenInLikeOpInternal}, + {"BetweenInLikeOpExpression", &PEGTransformerFactory::TransformBetweenInLikeOpExpressionInternal}, + {"LikeClause", &PEGTransformerFactory::TransformLikeClauseInternal}, + {"EscapeClause", &PEGTransformerFactory::TransformEscapeClauseInternal}, + {"LikeVariations", &PEGTransformerFactory::TransformLikeVariationsInternal}, + {"LikeToken", &PEGTransformerFactory::TransformLikeTokenInternal}, + {"ILikeToken", &PEGTransformerFactory::TransformILikeTokenInternal}, + {"GlobToken", &PEGTransformerFactory::TransformGlobTokenInternal}, + {"SimilarToToken", &PEGTransformerFactory::TransformSimilarToTokenInternal}, + {"NotILikeOp", &PEGTransformerFactory::TransformNotILikeOpInternal}, + {"NotLikeOp", &PEGTransformerFactory::TransformNotLikeOpInternal}, + {"NotSimilarToOp", &PEGTransformerFactory::TransformNotSimilarToOpInternal}, + {"InClause", &PEGTransformerFactory::TransformInClauseInternal}, + {"InExpression", &PEGTransformerFactory::TransformInExpressionInternal}, + {"InContainsExpression", &PEGTransformerFactory::TransformInContainsExpressionInternal}, + {"InExpressionList", &PEGTransformerFactory::TransformInExpressionListInternal}, + {"InSelectStatement", &PEGTransformerFactory::TransformInSelectStatementInternal}, + {"BetweenClause", &PEGTransformerFactory::TransformBetweenClauseInternal}, + {"OtherOperatorExpression", &PEGTransformerFactory::TransformOtherOperatorExpressionInternal}, + {"OtherOperatorTail", &PEGTransformerFactory::TransformOtherOperatorTailInternal}, + {"OtherOperator", &PEGTransformerFactory::TransformOtherOperatorInternal}, + {"AnyAllOperator", &PEGTransformerFactory::TransformAnyAllOperatorInternal}, + {"AnyOrAll", &PEGTransformerFactory::TransformAnyOrAllInternal}, + {"SubqueryAny", &PEGTransformerFactory::TransformSubqueryAnyInternal}, + {"SubqueryAll", &PEGTransformerFactory::TransformSubqueryAllInternal}, + {"InetOperator", &PEGTransformerFactory::TransformInetOperatorInternal}, + {"JsonOperator", &PEGTransformerFactory::TransformJsonOperatorInternal}, + {"ListOperator", &PEGTransformerFactory::TransformListOperatorInternal}, + {"StringOperator", &PEGTransformerFactory::TransformStringOperatorInternal}, + {"QualifiedOperator", &PEGTransformerFactory::TransformQualifiedOperatorInternal}, + {"QualifiedOperatorContents", &PEGTransformerFactory::TransformQualifiedOperatorContentsInternal}, + {"AnyOp", &PEGTransformerFactory::TransformAnyOpInternal}, + {"BitwiseExpression", &PEGTransformerFactory::TransformBitwiseExpressionInternal}, + {"BitwiseExpressionTail", &PEGTransformerFactory::TransformBitwiseExpressionTailInternal}, + {"BitOperator", &PEGTransformerFactory::TransformBitOperatorInternal}, + {"AdditiveExpression", &PEGTransformerFactory::TransformAdditiveExpressionInternal}, + {"AdditiveExpressionTail", &PEGTransformerFactory::TransformAdditiveExpressionTailInternal}, + {"Term", &PEGTransformerFactory::TransformTermInternal}, + {"MultiplicativeExpression", &PEGTransformerFactory::TransformMultiplicativeExpressionInternal}, + {"MultiplicativeExpressionTail", &PEGTransformerFactory::TransformMultiplicativeExpressionTailInternal}, + {"Factor", &PEGTransformerFactory::TransformFactorInternal}, + {"ExponentiationExpression", &PEGTransformerFactory::TransformExponentiationExpressionInternal}, + {"ExponentiationExpressionTail", &PEGTransformerFactory::TransformExponentiationExpressionTailInternal}, + {"ExponentOperator", &PEGTransformerFactory::TransformExponentOperatorInternal}, + {"CollateExpression", &PEGTransformerFactory::TransformCollateExpressionInternal}, + {"CollateExpressionTail", &PEGTransformerFactory::TransformCollateExpressionTailInternal}, + {"AtTimeZoneExpression", &PEGTransformerFactory::TransformAtTimeZoneExpressionInternal}, + {"AtTimeZoneExpressionTail", &PEGTransformerFactory::TransformAtTimeZoneExpressionTailInternal}, + {"PrefixOperator", &PEGTransformerFactory::TransformPrefixOperatorInternal}, + {"MinusPrefixOperator", &PEGTransformerFactory::TransformMinusPrefixOperatorInternal}, + {"PlusPrefixOperator", &PEGTransformerFactory::TransformPlusPrefixOperatorInternal}, + {"TildePrefixOperator", &PEGTransformerFactory::TransformTildePrefixOperatorInternal}, + {"BaseExpression", &PEGTransformerFactory::TransformBaseExpressionInternal}, + {"IndirectionList", &PEGTransformerFactory::TransformIndirectionListInternal}, + {"Indirection", &PEGTransformerFactory::TransformIndirectionInternal}, + {"CastOperator", &PEGTransformerFactory::TransformCastOperatorInternal}, + {"DotOperator", &PEGTransformerFactory::TransformDotOperatorInternal}, + {"DotMethodOperator", &PEGTransformerFactory::TransformDotMethodOperatorInternal}, + {"DotColumnOperator", &PEGTransformerFactory::TransformDotColumnOperatorInternal}, + {"MethodExpression", &PEGTransformerFactory::TransformMethodExpressionInternal}, + {"MethodExpressionArguments", &PEGTransformerFactory::TransformMethodExpressionArgumentsInternal}, + {"MethodExpressionArgumentList", &PEGTransformerFactory::TransformMethodExpressionArgumentListInternal}, + {"MethodFunctionArguments", &PEGTransformerFactory::TransformMethodFunctionArgumentsInternal}, + {"SliceExpression", &PEGTransformerFactory::TransformSliceExpressionInternal}, + {"SliceBound", &PEGTransformerFactory::TransformSliceBoundInternal}, + {"EndSliceBound", &PEGTransformerFactory::TransformEndSliceBoundInternal}, + {"EndSliceValue", &PEGTransformerFactory::TransformEndSliceValueInternal}, + {"EndSliceMinus", &PEGTransformerFactory::TransformEndSliceMinusInternal}, + {"StepSliceBound", &PEGTransformerFactory::TransformStepSliceBoundInternal}, + {"PostfixOperator", &PEGTransformerFactory::TransformPostfixOperatorInternal}, + {"SpecialFunctionExpression", &PEGTransformerFactory::TransformSpecialFunctionExpressionInternal}, + {"CoalesceExpression", &PEGTransformerFactory::TransformCoalesceExpressionInternal}, + {"UnpackExpression", &PEGTransformerFactory::TransformUnpackExpressionInternal}, + {"TryExpression", &PEGTransformerFactory::TransformTryExpressionInternal}, + {"ColumnsExpression", &PEGTransformerFactory::TransformColumnsExpressionInternal}, + {"ExtractExpression", &PEGTransformerFactory::TransformExtractExpressionInternal}, + {"ExtractArguments", &PEGTransformerFactory::TransformExtractArgumentsInternal}, + {"LambdaExpression", &PEGTransformerFactory::TransformLambdaExpressionInternal}, + {"NullIfExpression", &PEGTransformerFactory::TransformNullIfExpressionInternal}, + {"NullIfArguments", &PEGTransformerFactory::TransformNullIfArgumentsInternal}, + {"PositionExpression", &PEGTransformerFactory::TransformPositionExpressionInternal}, + {"PositionArguments", &PEGTransformerFactory::TransformPositionArgumentsInternal}, + {"RowExpression", &PEGTransformerFactory::TransformRowExpressionInternal}, + {"SubstringExpression", &PEGTransformerFactory::TransformSubstringExpressionInternal}, + {"SubstringArguments", &PEGTransformerFactory::TransformSubstringArgumentsInternal}, + {"SubstringExpressionList", &PEGTransformerFactory::TransformSubstringExpressionListInternal}, + {"SubstringParameters", &PEGTransformerFactory::TransformSubstringParametersInternal}, + {"SubstringFromFor", &PEGTransformerFactory::TransformSubstringFromForInternal}, + {"SubstringFromOptionalFor", &PEGTransformerFactory::TransformSubstringFromOptionalForInternal}, + {"SubstringFor", &PEGTransformerFactory::TransformSubstringForInternal}, + {"TrimExpression", &PEGTransformerFactory::TransformTrimExpressionInternal}, + {"TrimArguments", &PEGTransformerFactory::TransformTrimArgumentsInternal}, + {"TrimDirection", &PEGTransformerFactory::TransformTrimDirectionInternal}, + {"TrimBoth", &PEGTransformerFactory::TransformTrimBothInternal}, + {"TrimLeading", &PEGTransformerFactory::TransformTrimLeadingInternal}, + {"TrimTrailing", &PEGTransformerFactory::TransformTrimTrailingInternal}, + {"TrimSource", &PEGTransformerFactory::TransformTrimSourceInternal}, + {"OverlayExpression", &PEGTransformerFactory::TransformOverlayExpressionInternal}, + {"OverlayArguments", &PEGTransformerFactory::TransformOverlayArgumentsInternal}, + {"OverlayParameters", &PEGTransformerFactory::TransformOverlayParametersInternal}, + {"FromExpression", &PEGTransformerFactory::TransformFromExpressionInternal}, + {"ForExpression", &PEGTransformerFactory::TransformForExpressionInternal}, + {"OverlayExpressionList", &PEGTransformerFactory::TransformOverlayExpressionListInternal}, + {"ExtractArgument", &PEGTransformerFactory::TransformExtractArgumentInternal}, + {"ExtractDatePartArgument", &PEGTransformerFactory::TransformExtractDatePartArgumentInternal}, + {"ExtractIdentifierArgument", &PEGTransformerFactory::TransformExtractIdentifierArgumentInternal}, + {"ExtractStringArgument", &PEGTransformerFactory::TransformExtractStringArgumentInternal}, + {"ExtractDatePart", &PEGTransformerFactory::TransformExtractDatePartInternal}, {"InsertStatement", &PEGTransformerFactory::TransformInsertStatementInternal}, {"OrAction", &PEGTransformerFactory::TransformOrActionInternal}, {"InsertOrReplace", &PEGTransformerFactory::TransformInsertOrReplaceInternal}, diff --git a/src/duckdb/src/parser/peg/transformer/transform_select.cpp b/src/duckdb/src/parser/peg/transformer/transform_select.cpp index bf0751c66..f49eceea2 100644 --- a/src/duckdb/src/parser/peg/transformer/transform_select.cpp +++ b/src/duckdb/src/parser/peg/transformer/transform_select.cpp @@ -161,20 +161,6 @@ SetOperationType PEGTransformerFactory::TransformSetopExcept(PEGTransformer &tra return SetOperationType::EXCEPT; } -bool PEGTransformerFactory::TransformDistinctOrAll(PEGTransformer &transformer, ParseResult &parse_result) { - auto &list_pr = parse_result.Cast(); - auto &choice_pr = list_pr.Child(0); - return transformer.Transform(choice_pr.GetResult()); -} - -bool PEGTransformerFactory::TransformDistinctKeyword(PEGTransformer &transformer, ParseResult &parse_result) { - return true; -} - -bool PEGTransformerFactory::TransformAllKeyword(PEGTransformer &transformer, ParseResult &parse_result) { - return false; -} - bool PEGTransformerFactory::TransformLateral(PEGTransformer &transformer) { return true; } diff --git a/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp b/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp index 5de015273..f9a1fdef2 100644 --- a/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp @@ -1,3 +1,4 @@ +#include "duckdb/common/enum_util.hpp" #include "duckdb/common/limits.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/common/exception/parser_exception.hpp" @@ -18,6 +19,7 @@ #include "duckdb/parser/tableref/joinref.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/expression/bound_aggregate_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_expanded_expression.hpp" #include "duckdb/planner/expression_binder/column_alias_binder.hpp" @@ -200,36 +202,38 @@ void Binder::PrepareModifiers(OrderBinder &order_binder, QueryNode &statement, B } } } -#if 0 - // When this verification is enabled, replace ORDER BY x, y with ORDER BY create_sort_key(x, y) - // note that we don't enable this during actual verification since it doesn't always work - // e.g. it breaks EXPLAIN output on queries - bool can_replace = true; - for (auto &order_node : order.orders) { - if (order_node.expression->GetExpressionType() == ExpressionType::VALUE_CONSTANT) { - // we cannot replace the sort key when we order by literals (e.g. ORDER BY 1, 2` + if (config.options.debug_order_verification == DebugOrderVerification::CREATE_SORT_KEY) { + // When this verification is enabled, replace ORDER BY x, y with ORDER BY create_sort_key(x, y) + // note that we don't enable this during actual verification since it doesn't always work + // e.g. it breaks EXPLAIN output on queries + bool can_replace = true; + for (auto &order_node : order.orders) { + if (order_node.expression->GetExpressionType() == ExpressionType::VALUE_CONSTANT) { + // we cannot replace the sort key when we order by literals (e.g. ORDER BY 1, 2` + can_replace = false; + break; + } + } + if (!order_binder.HasExtraList()) { + // we can only do the replacement when we can order by elements that are not in the selection list can_replace = false; - break; } - } - if (!order_binder.HasExtraList()) { - // we can only do the replacement when we can order by elements that are not in the selection list - can_replace = false; - } - if (can_replace) { - vector> sort_key_parameters; - for (auto &order_node : order.orders) { - sort_key_parameters.push_back(std::move(order_node.expression)); - auto type = config.ResolveOrder(context, order_node.type); - auto null_order = config.ResolveNullOrder(context, type, order_node.null_order); - string sort_param = EnumUtil::ToString(type) + " " + EnumUtil::ToString(null_order); - sort_key_parameters.push_back(make_uniq(Value(sort_param))); + if (can_replace) { + vector> sort_key_parameters; + for (auto &order_node : order.orders) { + sort_key_parameters.push_back(std::move(order_node.expression)); + auto type = config.ResolveOrder(context, order_node.type); + auto null_order = config.ResolveNullOrder(context, type, order_node.null_order); + string sort_param = EnumUtil::ToString(type) + " " + EnumUtil::ToString(null_order); + sort_key_parameters.push_back(make_uniq(Value(sort_param))); + } + order.orders.clear(); + auto create_sort_key = + make_uniq("create_sort_key", std::move(sort_key_parameters)); + order.orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, + std::move(create_sort_key)); } - order.orders.clear(); - auto create_sort_key = make_uniq("create_sort_key", std::move(sort_key_parameters)); - order.orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, std::move(create_sort_key)); } -#endif for (auto &order_node : order.orders) { vector> order_list; order_binders[0].get().ExpandStarExpression(std::move(order_node.expression), order_list); @@ -378,9 +382,19 @@ void Binder::BindModifiers(BoundQueryNode &result, TableIndex table_index, const order.orders.emplace_back(order_type, null_order, std::move(expr)); } } + auto &config = DBConfig::GetConfig(context); for (auto &order_node : order.orders) { auto &expr = order_node.expression; - ExpressionBinder::PushCollation(context, order_node.expression, expr->GetReturnType()); + if (config.options.debug_order_verification == DebugOrderVerification::VARIANT && + expr->GetReturnType().id() != LogicalTypeId::VARIANT) { + // when this verification is enabled, cast every ORDER BY expression to VARIANT (the + // PushCollation below then routes ordering through the VARIANT comparator) to verify it + // produces the same ordering as the regular comparison. We do this on the *bound* + // expression so alias / positional / GROUP BY ALL resolution is unaffected. + order_node.expression = BoundCastExpression::AddCastToType( + context, std::move(order_node.expression), LogicalType::VARIANT()); + } + ExpressionBinder::PushCollation(context, order_node.expression, order_node.expression->GetReturnType()); } break; } diff --git a/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp b/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp index 3f6fe0a15..7197fb55e 100644 --- a/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +++ b/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp @@ -24,6 +24,32 @@ namespace duckdb { +static bool PlanReturnsExactlyOneRow(const LogicalOperator &op) { + switch (op.type) { + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: { + auto &aggr = op.Cast(); + return aggr.groups.empty() && aggr.grouping_sets.empty() && aggr.grouping_functions.empty(); + } + case LogicalOperatorType::LOGICAL_DUMMY_SCAN: + return true; + case LogicalOperatorType::LOGICAL_LIMIT: { + auto &limit = op.Cast(); + if (limit.limit_val.Type() != LimitNodeType::CONSTANT_VALUE || limit.limit_val.GetConstantValue() != 1) { + return false; + } + if (limit.offset_val.Type() != LimitNodeType::UNSET && + (limit.offset_val.Type() != LimitNodeType::CONSTANT_VALUE || limit.offset_val.GetConstantValue() != 0)) { + return false; + } + return op.children.size() == 1 && PlanReturnsExactlyOneRow(*op.children[0]); + } + case LogicalOperatorType::LOGICAL_PROJECTION: + return op.children.size() == 1 && PlanReturnsExactlyOneRow(*op.children[0]); + default: + return false; + } +} + static unique_ptr PlanUncorrelatedSubquery(Binder &binder, BoundSubqueryExpression &expr, unique_ptr &root, unique_ptr plan) { @@ -79,6 +105,11 @@ static unique_ptr PlanUncorrelatedSubquery(Binder &binder, BoundSubq // figure out the table index of the bound table of the entry which we want to return auto bindings = plan->GetColumnBindings(); D_ASSERT(bindings.size() == 1); + if (expr.GetReturnType().id() != LogicalTypeId::SQLNULL && PlanReturnsExactlyOneRow(*plan)) { + auto result = make_uniq(expr.GetName(), expr.GetReturnType(), bindings[0]); + root = LogicalCrossProduct::Create(std::move(root), std::move(plan)); + return std::move(result); + } auto table_idx = bindings[0].table_index; bool error_on_multiple_rows = Settings::Get(binder.context); diff --git a/src/duckdb/src/planner/collation_binding.cpp b/src/duckdb/src/planner/collation_binding.cpp index 8a4b2fc24..cedc1764d 100644 --- a/src/duckdb/src/planner/collation_binding.cpp +++ b/src/duckdb/src/planner/collation_binding.cpp @@ -141,9 +141,9 @@ bool PushVariantCollation(ClientContext &context, unique_ptr &source } auto &catalog = Catalog::GetSystemCatalog(context); auto &function_entry = - catalog.GetEntry(context, Identifier::DefaultSchema(), "variant_normalize"); + catalog.GetEntry(context, Identifier::DefaultSchema(), "variant_comparator"); if (function_entry.functions.Size() != 1) { - throw InternalException("variant_normalize should only have a single overload"); + throw InternalException("variant_comparator should only have a single overload"); } auto source_alias = source->GetAlias(); const auto &scalar_function = function_entry.functions.GetFunctionByOffset(0); diff --git a/src/duckdb/src/storage/compression/dict_fsst.cpp b/src/duckdb/src/storage/compression/dict_fsst.cpp index 5ad837e4d..085268647 100644 --- a/src/duckdb/src/storage/compression/dict_fsst.cpp +++ b/src/duckdb/src/storage/compression/dict_fsst.cpp @@ -190,12 +190,9 @@ static void DictFSSTFilter(ColumnSegment &segment, ColumnScanState &state, idx_t // apply the filter auto &dict_data = scan_state.dictionary->data; - UnifiedVectorFormat vdata; - dict_data.ToUnifiedFormat(vdata); SelectionVector dict_sel; idx_t filter_count = scan_state.dict_count; - ColumnSegment::FilterSelection(dict_sel, dict_data, vdata, filter, filter_state, scan_state.dict_count, - filter_count); + ColumnSegment::FilterSelection(dict_sel, dict_data, filter_state, scan_state.dict_count, filter_count); // now set all matching tuples to true for (idx_t i = 0; i < filter_count; i++) { @@ -225,10 +222,7 @@ static void DictFSSTFilter(ColumnSegment &segment, ColumnScanState &state, idx_t } // fallback: scan + filter DictFSSTCompressionStorage::StringScan(segment, state, vector_count, result); - - UnifiedVectorFormat vdata; - result.ToUnifiedFormat(vdata); - ColumnSegment::FilterSelection(sel, result, vdata, filter, filter_state, vector_count, sel_count); + ColumnSegment::FilterSelection(sel, result, filter_state, vector_count, sel_count); } } // namespace dict_fsst diff --git a/src/duckdb/src/storage/compression/rle.cpp b/src/duckdb/src/storage/compression/rle.cpp index da7bde835..fbaa93e2f 100644 --- a/src/duckdb/src/storage/compression/rle.cpp +++ b/src/duckdb/src/storage/compression/rle.cpp @@ -448,12 +448,9 @@ void RLEFilter(ColumnSegment &segment, ColumnScanState &state, idx_t vector_coun // execute the filter over all runs at once Vector run_vector(result.GetType(), data_ptr_cast(data_pointer), total_run_count); - UnifiedVectorFormat run_format; - run_vector.ToUnifiedFormat(run_format); - SelectionVector run_matches; scan_state.matching_run_count = total_run_count; - ColumnSegment::FilterSelection(run_matches, run_vector, run_format, filter, filter_state, total_run_count, + ColumnSegment::FilterSelection(run_matches, run_vector, filter_state, total_run_count, scan_state.matching_run_count); // for any runs that pass the filter - set the matches to true diff --git a/src/duckdb/src/storage/external_file_cache/caching_file_system.cpp b/src/duckdb/src/storage/external_file_cache/caching_file_system.cpp index 966ffa731..d5dc2f627 100644 --- a/src/duckdb/src/storage/external_file_cache/caching_file_system.cpp +++ b/src/duckdb/src/storage/external_file_cache/caching_file_system.cpp @@ -1,6 +1,7 @@ #include "duckdb/storage/external_file_cache/caching_file_system.hpp" #include "duckdb/common/checksum.hpp" +#include "duckdb/common/chrono.hpp" #include "duckdb/common/enums/cache_validation_mode.hpp" #include "duckdb/common/enums/memory_tag.hpp" #include "duckdb/common/file_system.hpp" @@ -66,7 +67,7 @@ class FetchBlockTask : public BaseExecutorTask { } const idx_t to_read = MinValue(block_size, file_size - offset); auto buf = buffer_manager.Allocate(MemoryTag::EXTERNAL_FILE_CACHE, to_read); - file_handle.Read(context, buf.GetDataMutable(), to_read, offset); + caching_file_handle.ReadAndRecord(context, buf.GetDataMutable(), to_read, offset); lk.lock(); block->block_handle = buf.GetBlockHandle(); @@ -256,9 +257,10 @@ FileBufferHandleGroup CachingFileHandle::Read(const idx_t nr_bytes, const idx_t return FileBufferHandleGroup(); } - if (!external_file_cache.IsEnabled()) { + // Uncached files are read directly into one contiguous buffer, skipping the per block syscalls and copies + if (!external_file_cache.IsEnabled() || !external_file_cache.ShouldCacheFile(path.path)) { auto buf = external_file_cache.GetBufferManager().Allocate(MemoryTag::EXTERNAL_FILE_CACHE, nr_bytes); - GetFileHandle().Read(context, buf.GetDataMutable(), nr_bytes, location); + ReadAndRecord(context, buf.GetDataMutable(), nr_bytes, location); vector mem_handles; mem_handles.push_back({std::move(buf), 0, nr_bytes}); return FileBufferHandleGroup(std::move(mem_handles)); @@ -376,6 +378,10 @@ string CachingFileHandle::GetVersionTag() { } bool CachingFileHandle::Validate() const { + if (!external_file_cache.ShouldCacheFile(path.path)) { + // uncached files have no stale state to serve, so their metadata must always be read fresh + return true; + } switch (validate) { case CacheValidationMode::VALIDATE_ALL: return true; @@ -410,8 +416,59 @@ bool CachingFileHandle::OnDiskFile() { return GetFileHandle().OnDiskFile(); } +void ReadThroughputEstimator::AddSample(double seconds, idx_t bytes) { + const double b = static_cast(bytes); + lock_guard guard(lock); + sample_count++; + sum_bytes += b; + sum_seconds += seconds; + sum_bytes_sq += b * b; + sum_bytes_seconds += b * seconds; +} + +bool ReadThroughputEstimator::TryEstimate(NetworkThroughputEstimate &result) const { + lock_guard guard(lock); + if (sample_count < 2) { + return false; + } + const double n = static_cast(sample_count); + const double variance = sum_bytes_sq - sum_bytes * sum_bytes / n; + if (!(variance > 0)) { + // every read was the same size, so we cant really get the latency + return false; + } + // least squares estimate of the line "total_time = intercept + slope * bytes". + const double slope = (sum_bytes_seconds - sum_bytes * sum_seconds / n) / variance; + const double intercept = (sum_seconds - slope * sum_bytes) / n; + if (!(slope > 0) || !(intercept > 0)) { + return false; + } + // intercept is the fixed start-up cost of a read, slope is seconds per byte so bandwidth is 1 / slope + result.latency_seconds = intercept; + result.bandwidth_bytes_per_s = 1.0 / slope; + return true; +} + bool CachingFileHandle::TryGetNetworkThroughput(NetworkThroughputEstimate &result) { - return GetFileHandle().TryGetNetworkThroughput(result); + // Remote files measure throughput in their own file system; local files fit it from this handle's own reads. + if (GetFileHandle().TryGetNetworkThroughput(result)) { + return true; + } + return throughput_estimator.TryEstimate(result); +} + +void CachingFileHandle::ReadAndRecord(QueryContext context, data_ptr_t buffer, idx_t nr_bytes, idx_t location) { + auto &handle = GetFileHandle(); + const auto read_start = steady_clock::now(); + handle.Read(context, buffer, nr_bytes, location); + RecordReadThroughput(duration(steady_clock::now() - read_start).count(), nr_bytes); +} + +void CachingFileHandle::RecordReadThroughput(double total_seconds, idx_t bytes) { + if (IsRemoteFile() || !(total_seconds > 0)) { + return; + } + throughput_estimator.AddSample(total_seconds, bytes); } idx_t CachingFileHandle::SeekPosition() { diff --git a/src/duckdb/src/storage/external_file_cache/external_file_cache.cpp b/src/duckdb/src/storage/external_file_cache/external_file_cache.cpp index 87cf77882..300bcbcdf 100644 --- a/src/duckdb/src/storage/external_file_cache/external_file_cache.cpp +++ b/src/duckdb/src/storage/external_file_cache/external_file_cache.cpp @@ -54,6 +54,15 @@ idx_t ExternalFileCache::GetCacheBlockSize(const string &path) const { return Settings::Get(db); } +bool ExternalFileCache::ShouldCacheFile(const string &path) const { + if (FileSystem::IsRemoteFile(path)) { + return true; + } + // Local files are not cached: the OS page cache already serves repeated reads + auto &db = buffer_manager.GetDatabase(); + return Settings::Get(db); +} + void ExternalFileCache::ReindexCachedFileCore(CachedFile &cached_file, idx_t file_size, idx_t old_block_size, idx_t new_block_size) { D_ASSERT(old_block_size > 0); diff --git a/src/duckdb/src/storage/table/column_data.cpp b/src/duckdb/src/storage/table/column_data.cpp index 85a9622f3..14a3654d8 100644 --- a/src/duckdb/src/storage/table/column_data.cpp +++ b/src/duckdb/src/storage/table/column_data.cpp @@ -377,9 +377,7 @@ void ColumnData::Filter(TransactionData transaction, idx_t vector_index, ColumnS idx_t scan_count = Scan(transaction, vector_index, state, result); FlatVector::SetSize(result, count_t(scan_count)); - UnifiedVectorFormat vdata; - result.ToUnifiedFormat(vdata); - ColumnSegment::FilterSelection(sel, result, vdata, filter, filter_state, scan_count, s_count); + ColumnSegment::FilterSelection(sel, result, filter_state, scan_count, s_count); } void ColumnData::Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, diff --git a/src/duckdb/src/storage/table/column_segment.cpp b/src/duckdb/src/storage/table/column_segment.cpp index c3a0cf85b..6ec4cfe4a 100644 --- a/src/duckdb/src/storage/table/column_segment.cpp +++ b/src/duckdb/src/storage/table/column_segment.cpp @@ -475,6 +475,11 @@ idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &vector, Unifi const TableFilter &filter, TableFilterState &filter_state, idx_t scan_count, idx_t &approved_tuple_count) { (void)vdata; + return FilterSelection(sel, vector, filter_state, scan_count, approved_tuple_count); +} + +idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &vector, TableFilterState &filter_state, + idx_t scan_count, idx_t &approved_tuple_count) { auto &state = filter_state.Cast(); return ExecuteExpressionFilterSelection(sel, vector, state, scan_count, approved_tuple_count); } diff --git a/src/duckdb/src/storage/table/row_id_column_data.cpp b/src/duckdb/src/storage/table/row_id_column_data.cpp index b3ca2393c..566d6f9bb 100644 --- a/src/duckdb/src/storage/table/row_id_column_data.cpp +++ b/src/duckdb/src/storage/table/row_id_column_data.cpp @@ -95,9 +95,7 @@ void RowIdColumnData::Filter(TransactionData transaction, idx_t vector_index, Co } // Now apply the filter - UnifiedVectorFormat vdata; - result.ToUnifiedFormat(vdata); - ColumnSegment::FilterSelection(sel, result, vdata, filter, filter_state, count, count); + ColumnSegment::FilterSelection(sel, result, filter_state, count, count); } void RowIdColumnData::Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, diff --git a/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h b/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h index 97d827867..a0875a398 100644 --- a/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +++ b/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h @@ -7,6 +7,7 @@ #include // For std::size_t #include +#include #include // For std::make_signed // VS2012 doesn't support deleted functions. diff --git a/src/duckdb/third_party/tdigest/t_digest.hpp b/src/duckdb/third_party/tdigest/t_digest.hpp index a25668c08..7abd26ec0 100644 --- a/src/duckdb/third_party/tdigest/t_digest.hpp +++ b/src/duckdb/third_party/tdigest/t_digest.hpp @@ -204,6 +204,20 @@ class TDigest { return maxProcessed_; } + Value min() const { + return min_; + } + + Value max() const { + return max_; + } + + // override the tracked minimum/maximum - used when reconstructing a serialized digest + void setMinMax(Value min, Value max) { + min_ = min; + max_ = max; + } + inline void add(std::vector digests) { add(digests.cbegin(), digests.cend()); } diff --git a/src/duckdb/ub_src_common_types_variant.cpp b/src/duckdb/ub_src_common_types_variant.cpp index f608fdfd1..3ea81f77f 100644 --- a/src/duckdb/ub_src_common_types_variant.cpp +++ b/src/duckdb/ub_src_common_types_variant.cpp @@ -1,5 +1,7 @@ #include "src/common/types/variant/variant.cpp" +#include "src/common/types/variant/variant_iterator.cpp" + #include "src/common/types/variant/variant_value.cpp" #include "src/common/types/variant/variant_value_convert.cpp" diff --git a/src/duckdb/ub_src_function_scalar_variant.cpp b/src/duckdb/ub_src_function_scalar_variant.cpp index 4d9489559..74a7e8b9d 100644 --- a/src/duckdb/ub_src_function_scalar_variant.cpp +++ b/src/duckdb/ub_src_function_scalar_variant.cpp @@ -2,6 +2,8 @@ #include "src/function/scalar/variant/variant_bind_utils.cpp" +#include "src/function/scalar/variant/variant_comparator.cpp" + #include "src/function/scalar/variant/variant_exists.cpp" #include "src/function/scalar/variant/variant_extract.cpp"