feat(parquet): support vector type storage

docs/source/user_guide/data_types.rst

@@ -186,6 +186,20 @@ and `Arrow DataTypes <https://arrow.apache.org/docs/format/Columnar.html#data-ty
 
        The type can be declared using ``ARRAY<t>`` where t is the data type of the contained elements.
 
+   * - ``VECTOR<t, n>``
+     - FixedSizeList
+     - Data type of a dense vector containing exactly ``n`` elements of type ``t``.
+
+       ``n`` must be positive. ``t`` can be ``BOOLEAN``, ``TINYINT``,
+       ``SMALLINT``, ``INT``, ``BIGINT``, ``FLOAT``, or ``DOUBLE``. A VECTOR
+       value may be NULL, but its elements cannot be NULL.
+
+       Paimon C++ currently supports VECTOR columns in Parquet data files. They
+       use the standard Parquet LIST representation on disk and are restored
+       as Arrow ``FixedSizeList`` values on read. VECTOR columns cannot be
+       primary, partition, or bucket keys. Dedicated vector storage and Data
+       Evolution support are not included yet.
+
    * - ``MAP<kt, vt>``
      - Map
      - Data type of an associative array that maps keys (including NULL) to values (including NULL). A map cannot contain duplicate keys; each key can map to at most one value.

include/paimon/defs.h

@@ -50,6 +50,8 @@ enum class FieldType {
     STRUCT = 15,
     BLOB = 16,
     VARIANT = 17,
+    /// Fixed-length dense vector represented by Arrow FixedSizeList.
+    VECTOR = 18,
     UNKNOWN = 128,
 };
 

include/paimon/format/column_stats.h

@@ -33,8 +33,8 @@ namespace paimon {
 /// ColumnStats is an abstract base class that represents statistical information for data columns
 /// in Paimon tables. It provides min/max values and null count statistics
 ///
-/// Only primitive data types support min/max statistics. Nested types (arrays, maps, structs) only
-/// track null counts through `NestedColumnStats`.
+/// Only primitive data types support min/max statistics. Nested types (arrays, vectors, maps,
+/// structs) only track null counts through `NestedColumnStats`.
 ///
 /// @note This is an abstract base class. Use the static factory methods `CreateXXXColumnStats()` to
 /// create concrete instances for specific data types.
@@ -52,7 +52,7 @@ class PAIMON_EXPORT ColumnStats {
     /// @name CreateXXXColumnStats()
     /// %Factory methods `CreateXXXColumnStats()` to create column statistics.
     /// - min/max/null_count for primitive data types
-    /// - null_count for nested data types (arrays, maps, structs)
+    /// - null_count for nested data types (arrays, vectors, maps, structs)
     ///
     /// @{
     static std::unique_ptr<ColumnStats> CreateBooleanColumnStats(std::optional<bool> min,
@@ -88,8 +88,8 @@ class PAIMON_EXPORT ColumnStats {
     static std::unique_ptr<ColumnStats> CreateDateColumnStats(std::optional<int32_t> min,
                                                               std::optional<int32_t> max,
                                                               std::optional<int64_t> null_count);
-    /// Creates column statistics for nested data types (arrays, maps, structs), which only track
-    /// null counts.
+    /// Creates column statistics for nested data types (arrays, vectors, maps, structs), which only
+    /// track null counts.
     static std::unique_ptr<ColumnStats> CreateNestedColumnStats(const FieldType& nested_type,
                                                                 std::optional<int64_t> null_count);
     /// @}
@@ -180,7 +180,7 @@ class PAIMON_EXPORT NestedColumnStats : public ColumnStats {
     NestedColumnStats(const FieldType& nested_type, std::optional<int64_t> null_count)
         : nested_type_(nested_type), null_count_(null_count) {
         assert(nested_type == FieldType::ARRAY || nested_type == FieldType::MAP ||
-               nested_type == FieldType::STRUCT);
+               nested_type == FieldType::STRUCT || nested_type == FieldType::VECTOR);
     }
 
     std::optional<int64_t> NullCount() const override {

src/paimon/common/predicate/literal_converter.cpp

@@ -166,6 +166,7 @@ Result<Literal> LiteralConverter::ConvertLiteralsFromRow(
         case FieldType::DATE:
             return Literal(FieldType::DATE, row.GetInt(field_idx));
         case FieldType::ARRAY:
+        case FieldType::VECTOR:
         case FieldType::MAP:
         case FieldType::STRUCT:
         default:

src/paimon/common/types/data_type.cpp

@@ -29,6 +29,7 @@
 #include "paimon/common/types/array_type.h"
 #include "paimon/common/types/map_type.h"
 #include "paimon/common/types/row_type.h"
+#include "paimon/common/types/vector_type.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/decimal_utils.h"
@@ -52,6 +53,8 @@ std::unique_ptr<DataType> DataType::Create(
             return std::make_unique<MapType>(type, nullable, metadata);
         case arrow::Type::type::LIST:
             return std::make_unique<ArrayType>(type, nullable, metadata);
+        case arrow::Type::type::FIXED_SIZE_LIST:
+            return std::make_unique<VectorType>(type, nullable, metadata);
         case arrow::Type::type::STRUCT:
             if (VariantTypeUtils::IsVariantMetadata(metadata)) {
                 // A variant field is physically a struct<value, metadata> but is a scalar

src/paimon/common/types/data_type_json_parser.cpp

@@ -21,18 +21,21 @@
 
 #include <algorithm>
 #include <cctype>
+#include <charconv>
 #include <cstddef>
 #include <cstdint>
 #include <limits>
 #include <map>
 #include <sstream>
+#include <system_error>
 #include <utility>
 #include <vector>
 
 #include "fmt/format.h"
 #include "paimon/common/data/blob_utils.h"
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/types/vector_type.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/rapidjson_util.h"
 #include "paimon/common/utils/string_utils.h"
@@ -148,6 +151,7 @@ enum class Keyword : int32_t {
     ROW,
     BLOB,
     VARIANT,
+    VECTOR,
     // NULL is keyword in c++
     NULL_,
     RAW,
@@ -197,6 +201,7 @@ const std::map<std::string, Keyword>& Keywords() {
         {"ROW", Keyword::ROW},
         {"BLOB", Keyword::BLOB},
         {"VARIANT", Keyword::VARIANT},
+        {"VECTOR", Keyword::VECTOR},
         {"NULL", Keyword::NULL_},
         {"RAW", Keyword::RAW},
         {"LEGACY", Keyword::LEGACY},
@@ -249,6 +254,7 @@ class TokenParser {
     Result<std::shared_ptr<arrow::DataType>> ParseDoubleType();
     Result<std::shared_ptr<arrow::DataType>> ParseTimestampType();
     Result<std::shared_ptr<arrow::DataType>> ParseTimestampLtzType();
+    Result<std::shared_ptr<arrow::DataType>> ParseVectorType();
     Result<int32_t> ParseOptionalPrecision(int32_t default_precision);
 
  private:
@@ -526,6 +532,8 @@ Result<std::shared_ptr<arrow::DataType>> TokenParser::ParseTypeByKeyword(
             return ParseTimestampType();
         case Keyword::TIMESTAMP_LTZ:
             return ParseTimestampLtzType();
+        case Keyword::VECTOR:
+            return ParseVectorType();
         default:
             return Status::Invalid(fmt::format("Unsupported type: {}", GetToken().value));
     }
@@ -607,6 +615,34 @@ Result<std::shared_ptr<arrow::DataType>> TokenParser::ParseTimestampLtzType() {
     return ts_type;
 }
 
+Result<std::shared_ptr<arrow::DataType>> TokenParser::ParseVectorType() {
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::BEGIN_SUBTYPE));
+    bool element_nullable = true;
+    AtomicTypeAttributes element_attributes;
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::DataType> element_type,
+                           ParseTypeWithNullability(&element_nullable, &element_attributes));
+    if (element_attributes.is_blob || element_attributes.is_variant ||
+        !VectorType::IsValidElementType(element_type)) {
+        return Status::Invalid(
+            fmt::format("Invalid element type for vector: {}", element_type->ToString()));
+    }
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::LIST_SEPARATOR));
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::LITERAL_INT));
+    const std::string& length_token = GetToken().value;
+    int64_t length = 0;
+    const auto [end, error] =
+        std::from_chars(length_token.data(), length_token.data() + length_token.size(), length);
+    if (error != std::errc() || end != length_token.data() + length_token.size() || length < 1 ||
+        length > std::numeric_limits<int32_t>::max()) {
+        return Status::Invalid(
+            fmt::format("Vector length must be between 1 and {} (both inclusive), but was {}",
+                        std::numeric_limits<int32_t>::max(), length_token));
+    }
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::END_SUBTYPE));
+    return arrow::fixed_size_list(arrow::field("item", element_type, element_nullable),
+                                  static_cast<int32_t>(length));
+}
+
 Result<int32_t> TokenParser::ParseOptionalPrecision(int32_t default_precision) {
     auto precision = default_precision;
     if (HasNextToken({TokenType::BEGIN_PARAMETER})) {
@@ -659,6 +695,8 @@ Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseComplexTypeField(
 
     if (StringUtils::StartsWith(type_str, "ARRAY")) {
         return ParseArrayType(name, type_json_value, nullable);
+    } else if (StringUtils::StartsWith(type_str, "VECTOR")) {
+        return ParseVectorType(name, type_json_value, nullable);
     } else if (StringUtils::StartsWith(type_str, "MAP")) {
         return ParseMapType(name, type_json_value, nullable);
     } else if (StringUtils::StartsWith(type_str, "ROW")) {
@@ -681,6 +719,27 @@ Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseArrayType(
     return arrow::field(name, arrow::list(element_field), nullable);
 }
 
+Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseVectorType(
+    const std::string& name, const rapidjson::Value& type_json_value, bool nullable) {
+    if (!type_json_value.HasMember("element") || !type_json_value.HasMember("length")) {
+        return Status::Invalid("vector data type must have element and length");
+    }
+    if (!type_json_value["length"].IsInt()) {
+        return Status::Invalid("vector length must be an integer");
+    }
+    int32_t length = type_json_value["length"].GetInt();
+    if (length < 1) {
+        return Status::Invalid("Vector length must be between 1 and 2147483647 (both inclusive)");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Field> element_field,
+                           ParseType("item", type_json_value["element"]));
+    if (!VectorType::IsValidElementType(element_field->type())) {
+        return Status::Invalid(
+            fmt::format("Invalid element type for vector: {}", element_field->type()->ToString()));
+    }
+    return arrow::field(name, arrow::fixed_size_list(element_field, length), nullable);
+}
+
 Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseMapType(
     const std::string& name, const rapidjson::Value& type_json_value, bool nullable) {
     if (!type_json_value.HasMember("key") || !type_json_value.HasMember("value")) {

src/paimon/common/types/data_type_json_parser.h

@@ -50,6 +50,8 @@ class DataTypeJsonParser {
 
     static Result<std::shared_ptr<arrow::Field>> ParseArrayType(
         const std::string& name, const rapidjson::Value& type_json_value, bool nullable);
+    static Result<std::shared_ptr<arrow::Field>> ParseVectorType(
+        const std::string& name, const rapidjson::Value& type_json_value, bool nullable);
     static Result<std::shared_ptr<arrow::Field>> ParseMapType(
         const std::string& name, const rapidjson::Value& type_json_value, bool nullable);
     static Result<std::shared_ptr<arrow::Field>> ParseRowType(

src/paimon/common/types/data_type_json_parser_test.cpp

@@ -48,6 +48,56 @@ TEST(DataTypeJsonParserTest, ParseTypeArrayTypeSuccess) {
     ASSERT_NE(field, nullptr);
 }
 
+TEST(DataTypeJsonParserTest, ParseVectorTypeSuccess) {
+    const char* json = R"({
+        "type": "VECTOR NOT NULL",
+        "element": "FLOAT",
+        "length": 3
+    })";
+    rapidjson::Document doc;
+    doc.Parse(json);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Field> field,
+                         DataTypeJsonParser::ParseType("embedding", doc));
+    ASSERT_FALSE(field->nullable());
+    ASSERT_EQ(field->type()->id(), arrow::Type::FIXED_SIZE_LIST);
+    auto vector_type = std::static_pointer_cast<arrow::FixedSizeListType>(field->type());
+    ASSERT_EQ(vector_type->list_size(), 3);
+    ASSERT_TRUE(vector_type->value_type()->Equals(arrow::float32()));
+
+    rapidjson::Document sql_doc;
+    rapidjson::Value sql_value("VECTOR<BIGINT NOT NULL, 5>", sql_doc.GetAllocator());
+    ASSERT_OK_AND_ASSIGN(field, DataTypeJsonParser::ParseType("embedding", sql_value));
+    vector_type = std::static_pointer_cast<arrow::FixedSizeListType>(field->type());
+    ASSERT_TRUE(field->nullable());
+    ASSERT_EQ(vector_type->list_size(), 5);
+    ASSERT_FALSE(vector_type->value_field()->nullable());
+    ASSERT_TRUE(vector_type->value_type()->Equals(arrow::int64()));
+}
+
+TEST(DataTypeJsonParserTest, ParseVectorTypeFailure) {
+    for (const char* json : {
+             R"({"type":"VECTOR","element":"FLOAT","length":0})",
+             R"({"type":"VECTOR","element":"STRING","length":3})",
+             R"({"type":"VECTOR","element":"FLOAT"})",
+             R"({"type":"VECTOR","element":"FLOAT","length":"3"})",
+         }) {
+        rapidjson::Document doc;
+        doc.Parse(json);
+        ASSERT_NOK(DataTypeJsonParser::ParseType("embedding", doc));
+    }
+
+    rapidjson::Document sql_doc;
+    rapidjson::Value sql_value("VECTOR<STRING, 3>", sql_doc.GetAllocator());
+    ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("embedding", sql_value),
+                        "Invalid element type for vector");
+    sql_value.SetString("VECTOR<DOUBLE, 3>", sql_doc.GetAllocator());
+    ASSERT_OK(DataTypeJsonParser::ParseType("embedding", sql_value));
+    sql_value.SetString("VECTOR<FLOAT, 999999999999999999999999>", sql_doc.GetAllocator());
+    ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("embedding", sql_value),
+                        "Vector length must be between 1 and 2147483647");
+}
+
 TEST(DataTypeJsonParserTest, ParseTypeMapTypeSuccess) {
     const std::string name = "map_field";
     const char* json = R"({

src/paimon/common/types/data_type_test.cpp

@@ -148,4 +148,18 @@ TEST(DataTypeTest, NestedTypeSerializationUsesChildMetadata) {
               R"({"type":"ARRAY","element":"INT"})");
 }
 
+TEST(DataTypeTest, VectorTypeSerialization) {
+    auto vector_field = arrow::field(
+        "embedding", arrow::fixed_size_list(arrow::field("item", arrow::float32()), 3), false);
+    auto data_type =
+        DataType::Create(vector_field->type(), vector_field->nullable(), vector_field->metadata());
+    rapidjson::Document doc;
+    auto value = data_type->ToJson(&doc.GetAllocator());
+    rapidjson::StringBuffer buffer;
+    rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+    value.Accept(writer);
+    ASSERT_EQ(std::string(buffer.GetString()),
+              R"({"type":"VECTOR NOT NULL","element":"FLOAT","length":3})");
+}
+
 }  // namespace paimon::test

src/paimon/common/types/vector_type.h

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "arrow/api.h"
+#include "paimon/common/types/data_type.h"
+#include "paimon/common/utils/rapidjson_util.h"
+
+namespace paimon {
+
+/// Fixed-size VECTOR<T, N> logical type backed by Arrow FixedSizeList.
+class VectorType : public DataType {
+ public:
+    static constexpr char TYPE[] = "VECTOR";
+
+    VectorType(const std::shared_ptr<arrow::DataType>& type, bool nullable,
+               const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+        : DataType(type, nullable, metadata) {}
+
+    static bool IsValidElementType(const std::shared_ptr<arrow::DataType>& type) {
+        switch (type->id()) {
+            case arrow::Type::BOOL:
+            case arrow::Type::INT8:
+            case arrow::Type::INT16:
+            case arrow::Type::INT32:
+            case arrow::Type::INT64:
+            case arrow::Type::FLOAT:
+            case arrow::Type::DOUBLE:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const
+        noexcept(false) override {
+        rapidjson::Value obj(rapidjson::kObjectType);
+        obj.AddMember(
+            rapidjson::StringRef("type"),
+            RapidJsonUtil::SerializeValue(WithNullable(std::string(TYPE)), allocator).Move(),
+            *allocator);
+        auto* type = static_cast<arrow::FixedSizeListType*>(type_.get());
+        auto value_field = type->value_field();
+        std::shared_ptr<DataType> data_type =
+            DataType::Create(value_field->type(), value_field->nullable(), value_field->metadata());
+        obj.AddMember(rapidjson::StringRef("element"),
+                      RapidJsonUtil::SerializeValue(*data_type, allocator).Move(), *allocator);
+        obj.AddMember(rapidjson::StringRef("length"),
+                      RapidJsonUtil::SerializeValue(type->list_size(), allocator).Move(),
+                      *allocator);
+        return obj;
+    }
+};
+
+}  // namespace paimon