https://github.com/ceph/s3select/pull/169 From 8da50fe3138e72831efd935d0d41c329974ce760 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 6 May 2025 14:59:48 +0100 Subject: [PATCH] Refactor crypto integration with arrow arrow >= 20.0.0 refactored the crypto API to support better multithreading; align to new API with backwards compatibility. --- include/encryption_internal.h | 100 +----------------- include/encryption_internal_19.h | 114 +++++++++++++++++++++ include/encryption_internal_20.h | 141 +++++++++++++++++++++++++ include/internal_file_decryptor.h | 107 +------------------ include/internal_file_decryptor_19.h | 121 ++++++++++++++++++++++ include/internal_file_decryptor_20.h | 148 +++++++++++++++++++++++++++ include/s3select_parquet_intrf.h | 39 +++++++ 7 files changed, 573 insertions(+), 197 deletions(-) create mode 100644 include/encryption_internal_19.h create mode 100644 include/encryption_internal_20.h create mode 100644 include/internal_file_decryptor_19.h create mode 100644 include/internal_file_decryptor_20.h diff --git a/include/encryption_internal.h b/include/encryption_internal.h index c8deee49..00129a2e 100644 --- a/src/s3select/include/encryption_internal.h +++ b/src/s3select/include/encryption_internal.h @@ -17,98 +17,8 @@ #pragma once -#include -#include -#include - -#include "parquet/properties.h" -#include "parquet/types.h" - -using parquet::ParquetCipher; - -namespace parquet { -namespace encryption { - -constexpr int kGcmTagLength = 16; -constexpr int kNonceLength = 12; - -// Module types -constexpr int8_t kFooter = 0; -constexpr int8_t kColumnMetaData = 1; -constexpr int8_t kDataPage = 2; -constexpr int8_t kDictionaryPage = 3; -constexpr int8_t kDataPageHeader = 4; -constexpr int8_t kDictionaryPageHeader = 5; -constexpr int8_t kColumnIndex = 6; -constexpr int8_t kOffsetIndex = 7; - -/// Performs AES encryption operations with GCM or CTR ciphers. -class AesEncryptor { - public: - static AesEncryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata, - std::vector* all_encryptors); - - ~AesEncryptor(); - - /// Size difference between plaintext and ciphertext, for this cipher. - int CiphertextSizeDelta(); - - /// Encrypts plaintext with the key and aad. Key length is passed only for validation. - /// If different from value in constructor, exception will be thrown. - int Encrypt(const uint8_t* plaintext, int plaintext_len, const uint8_t* key, - int key_len, const uint8_t* aad, int aad_len, uint8_t* ciphertext); - - /// Encrypts plaintext footer, in order to compute footer signature (tag). - int SignedFooterEncrypt(const uint8_t* footer, int footer_len, const uint8_t* key, - int key_len, const uint8_t* aad, int aad_len, - const uint8_t* nonce, uint8_t* encrypted_footer); - - void WipeOut(); - - private: - /// Can serve one key length only. Possible values: 16, 24, 32 bytes. - explicit AesEncryptor(ParquetCipher::type alg_id, int key_len, bool metadata); - // PIMPL Idiom - class AesEncryptorImpl; - std::unique_ptr impl_; -}; - -/// Performs AES decryption operations with GCM or CTR ciphers. -class AesDecryptor { - public: - static AesDecryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata, - std::vector* all_decryptors); - - ~AesDecryptor(); - void WipeOut(); - - /// Size difference between plaintext and ciphertext, for this cipher. - int CiphertextSizeDelta(); - - /// Decrypts ciphertext with the key and aad. Key length is passed only for - /// validation. If different from value in constructor, exception will be thrown. - int Decrypt(const uint8_t* ciphertext, int ciphertext_len, const uint8_t* key, - int key_len, const uint8_t* aad, int aad_len, uint8_t* plaintext); - - private: - /// Can serve one key length only. Possible values: 16, 24, 32 bytes. - explicit AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadata); - // PIMPL Idiom - class AesDecryptorImpl; - std::unique_ptr impl_; -}; - -std::string CreateModuleAad(const std::string& file_aad, int8_t module_type, - int16_t row_group_ordinal, int16_t column_ordinal, - int16_t page_ordinal); - -std::string CreateFooterAad(const std::string& aad_prefix_bytes); - -// Update last two bytes of page (or page header) module AAD -void QuickUpdatePageAad(const std::string& AAD, int16_t new_page_ordinal); - -// Wraps OpenSSL RAND_bytes function -void RandBytes(unsigned char* buf, int num); - -} // namespace encryption -} // namespace parquet +#if ARROW_VERSION_MAJOR < 20 +#include "encryption_internal_19.h" +#else +#include "encryption_internal_20.h" +#endif diff --git a/include/encryption_internal_19.h b/include/encryption_internal_19.h new file mode 100644 index 00000000..c8deee49 --- /dev/null +++ b/src/s3select/include/encryption_internal_19.h @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "parquet/properties.h" +#include "parquet/types.h" + +using parquet::ParquetCipher; + +namespace parquet { +namespace encryption { + +constexpr int kGcmTagLength = 16; +constexpr int kNonceLength = 12; + +// Module types +constexpr int8_t kFooter = 0; +constexpr int8_t kColumnMetaData = 1; +constexpr int8_t kDataPage = 2; +constexpr int8_t kDictionaryPage = 3; +constexpr int8_t kDataPageHeader = 4; +constexpr int8_t kDictionaryPageHeader = 5; +constexpr int8_t kColumnIndex = 6; +constexpr int8_t kOffsetIndex = 7; + +/// Performs AES encryption operations with GCM or CTR ciphers. +class AesEncryptor { + public: + static AesEncryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata, + std::vector* all_encryptors); + + ~AesEncryptor(); + + /// Size difference between plaintext and ciphertext, for this cipher. + int CiphertextSizeDelta(); + + /// Encrypts plaintext with the key and aad. Key length is passed only for validation. + /// If different from value in constructor, exception will be thrown. + int Encrypt(const uint8_t* plaintext, int plaintext_len, const uint8_t* key, + int key_len, const uint8_t* aad, int aad_len, uint8_t* ciphertext); + + /// Encrypts plaintext footer, in order to compute footer signature (tag). + int SignedFooterEncrypt(const uint8_t* footer, int footer_len, const uint8_t* key, + int key_len, const uint8_t* aad, int aad_len, + const uint8_t* nonce, uint8_t* encrypted_footer); + + void WipeOut(); + + private: + /// Can serve one key length only. Possible values: 16, 24, 32 bytes. + explicit AesEncryptor(ParquetCipher::type alg_id, int key_len, bool metadata); + // PIMPL Idiom + class AesEncryptorImpl; + std::unique_ptr impl_; +}; + +/// Performs AES decryption operations with GCM or CTR ciphers. +class AesDecryptor { + public: + static AesDecryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata, + std::vector* all_decryptors); + + ~AesDecryptor(); + void WipeOut(); + + /// Size difference between plaintext and ciphertext, for this cipher. + int CiphertextSizeDelta(); + + /// Decrypts ciphertext with the key and aad. Key length is passed only for + /// validation. If different from value in constructor, exception will be thrown. + int Decrypt(const uint8_t* ciphertext, int ciphertext_len, const uint8_t* key, + int key_len, const uint8_t* aad, int aad_len, uint8_t* plaintext); + + private: + /// Can serve one key length only. Possible values: 16, 24, 32 bytes. + explicit AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadata); + // PIMPL Idiom + class AesDecryptorImpl; + std::unique_ptr impl_; +}; + +std::string CreateModuleAad(const std::string& file_aad, int8_t module_type, + int16_t row_group_ordinal, int16_t column_ordinal, + int16_t page_ordinal); + +std::string CreateFooterAad(const std::string& aad_prefix_bytes); + +// Update last two bytes of page (or page header) module AAD +void QuickUpdatePageAad(const std::string& AAD, int16_t new_page_ordinal); + +// Wraps OpenSSL RAND_bytes function +void RandBytes(unsigned char* buf, int num); + +} // namespace encryption +} // namespace parquet diff --git a/include/encryption_internal_20.h b/include/encryption_internal_20.h new file mode 100644 index 00000000..06252749 --- /dev/null +++ b/src/s3select/include/encryption_internal_20.h @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/util/span.h" +#include "parquet/properties.h" +#include "parquet/types.h" + +using parquet::ParquetCipher; + +namespace parquet::encryption { + +constexpr int32_t kGcmTagLength = 16; +constexpr int32_t kNonceLength = 12; + +// Module types +constexpr int8_t kFooter = 0; +constexpr int8_t kColumnMetaData = 1; +constexpr int8_t kDataPage = 2; +constexpr int8_t kDictionaryPage = 3; +constexpr int8_t kDataPageHeader = 4; +constexpr int8_t kDictionaryPageHeader = 5; +constexpr int8_t kColumnIndex = 6; +constexpr int8_t kOffsetIndex = 7; +constexpr int8_t kBloomFilterHeader = 8; +constexpr int8_t kBloomFilterBitset = 9; + +/// Performs AES encryption operations with GCM or CTR ciphers. +class PARQUET_EXPORT AesEncryptor { + public: + /// Can serve one key length only. Possible values: 16, 24, 32 bytes. + /// If write_length is true, prepend ciphertext length to the ciphertext + explicit AesEncryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata, + bool write_length = true); + + static std::unique_ptr Make(ParquetCipher::type alg_id, int32_t key_len, + bool metadata, bool write_length = true); + + ~AesEncryptor(); + + /// The size of the ciphertext, for this cipher and the specified plaintext length. + [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const; + + /// Encrypts plaintext with the key and aad. Key length is passed only for validation. + /// If different from value in constructor, exception will be thrown. + int32_t Encrypt(::arrow::util::span plaintext, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span ciphertext); + + /// Encrypts plaintext footer, in order to compute footer signature (tag). + int32_t SignedFooterEncrypt(::arrow::util::span footer, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span nonce, + ::arrow::util::span encrypted_footer); + + private: + // PIMPL Idiom + class AesEncryptorImpl; + std::unique_ptr impl_; +}; + +/// Performs AES decryption operations with GCM or CTR ciphers. +class PARQUET_EXPORT AesDecryptor { + public: + /// \brief Construct an AesDecryptor + /// + /// \param alg_id the encryption algorithm to use + /// \param key_len key length. Possible values: 16, 24, 32 bytes. + /// \param metadata if true then this is a metadata decryptor + /// \param contains_length if true, expect ciphertext length prepended to the ciphertext + explicit AesDecryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata, + bool contains_length = true); + + static std::unique_ptr Make(ParquetCipher::type alg_id, int32_t key_len, + bool metadata); + + ~AesDecryptor(); + + /// The size of the plaintext, for this cipher and the specified ciphertext length. + [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const; + + /// The size of the ciphertext, for this cipher and the specified plaintext length. + [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const; + + /// Decrypts ciphertext with the key and aad. Key length is passed only for + /// validation. If different from value in constructor, exception will be thrown. + /// The caller is responsible for ensuring that the plaintext buffer is at least as + /// large as PlaintextLength(ciphertext_len). + int32_t Decrypt(::arrow::util::span ciphertext, + ::arrow::util::span key, + ::arrow::util::span aad, + ::arrow::util::span plaintext); + + private: + // PIMPL Idiom + class AesDecryptorImpl; + std::unique_ptr impl_; +}; + +std::string CreateModuleAad(const std::string& file_aad, int8_t module_type, + int16_t row_group_ordinal, int16_t column_ordinal, + int32_t page_ordinal); + +std::string CreateFooterAad(const std::string& aad_prefix_bytes); + +// Update last two bytes of page (or page header) module AAD +void QuickUpdatePageAad(int32_t new_page_ordinal, std::string* AAD); + +// Wraps OpenSSL RAND_bytes function +void RandBytes(unsigned char* buf, size_t num); + +// Ensure OpenSSL is initialized. +// +// This is only necessary in specific situations since OpenSSL otherwise +// initializes itself automatically. For example, under Valgrind, a memory +// leak will be reported if OpenSSL is initialized for the first time from +// a worker thread; calling this function from the main thread prevents this. +void EnsureBackendInitialized(); + +} // namespace parquet::encryption diff --git a/include/internal_file_decryptor.h b/include/internal_file_decryptor.h index 011c4acb..1e4f4b99 100644 --- a/src/s3select/include/internal_file_decryptor.h +++ b/src/s3select/include/internal_file_decryptor.h @@ -17,105 +17,8 @@ #pragma once -#include -#include -#include -#include - -#include "parquet/schema.h" - -namespace parquet { - -namespace encryption { -class AesDecryptor; -class AesEncryptor; -} // namespace encryption - -class FileDecryptionProperties; - -class PARQUET_EXPORT Decryptor { - public: - Decryptor(encryption::AesDecryptor* decryptor, const std::string& key, - const std::string& file_aad, const std::string& aad, - ::arrow::MemoryPool* pool); - - const std::string& file_aad() const { return file_aad_; } - void UpdateAad(const std::string& aad) { aad_ = aad; } - ::arrow::MemoryPool* pool() { return pool_; } - - int CiphertextSizeDelta(); - int Decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext); - - private: - encryption::AesDecryptor* aes_decryptor_; - std::string key_; - std::string file_aad_; - std::string aad_; - ::arrow::MemoryPool* pool_; -}; - -class InternalFileDecryptor { - public: - explicit InternalFileDecryptor(FileDecryptionProperties* properties, - const std::string& file_aad, - ParquetCipher::type algorithm, - const std::string& footer_key_metadata, - ::arrow::MemoryPool* pool); - - std::string& file_aad() { return file_aad_; } - - std::string GetFooterKey(); - - ParquetCipher::type algorithm() { return algorithm_; } - - std::string& footer_key_metadata() { return footer_key_metadata_; } - - FileDecryptionProperties* properties() { return properties_; } - - void WipeOutDecryptionKeys(); - - ::arrow::MemoryPool* pool() { return pool_; } - - std::shared_ptr GetFooterDecryptor(); - std::shared_ptr GetFooterDecryptorForColumnMeta(const std::string& aad = ""); - std::shared_ptr GetFooterDecryptorForColumnData(const std::string& aad = ""); - std::shared_ptr GetColumnMetaDecryptor( - const std::string& column_path, const std::string& column_key_metadata, - const std::string& aad = ""); - std::shared_ptr GetColumnDataDecryptor( - const std::string& column_path, const std::string& column_key_metadata, - const std::string& aad = ""); - - private: - FileDecryptionProperties* properties_; - // Concatenation of aad_prefix (if exists) and aad_file_unique - std::string file_aad_; - std::map> column_data_map_; - std::map> column_metadata_map_; - - std::shared_ptr footer_metadata_decryptor_; - std::shared_ptr footer_data_decryptor_; - ParquetCipher::type algorithm_; - std::string footer_key_metadata_; - std::vector all_decryptors_; - - /// Key must be 16, 24 or 32 bytes in length. Thus there could be up to three - // types of meta_decryptors and data_decryptors. - std::unique_ptr meta_decryptor_[3]; - std::unique_ptr data_decryptor_[3]; - - ::arrow::MemoryPool* pool_; - - std::shared_ptr GetFooterDecryptor(const std::string& aad, bool metadata); - std::shared_ptr GetColumnDecryptor(const std::string& column_path, - const std::string& column_key_metadata, - const std::string& aad, - bool metadata = false); - - encryption::AesDecryptor* GetMetaAesDecryptor(size_t key_size); - encryption::AesDecryptor* GetDataAesDecryptor(size_t key_size); - - int MapKeyLenToDecryptorArrayIndex(int key_len); -}; - -} // namespace parquet +#if ARROW_VERSION_MAJOR < 20 +#include "internal_file_decryptor_19.h" +#else +#include "internal_file_decryptor_20.h" +#endif diff --git a/include/internal_file_decryptor_19.h b/include/internal_file_decryptor_19.h new file mode 100644 index 00000000..011c4acb --- /dev/null +++ b/src/s3select/include/internal_file_decryptor_19.h @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "parquet/schema.h" + +namespace parquet { + +namespace encryption { +class AesDecryptor; +class AesEncryptor; +} // namespace encryption + +class FileDecryptionProperties; + +class PARQUET_EXPORT Decryptor { + public: + Decryptor(encryption::AesDecryptor* decryptor, const std::string& key, + const std::string& file_aad, const std::string& aad, + ::arrow::MemoryPool* pool); + + const std::string& file_aad() const { return file_aad_; } + void UpdateAad(const std::string& aad) { aad_ = aad; } + ::arrow::MemoryPool* pool() { return pool_; } + + int CiphertextSizeDelta(); + int Decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext); + + private: + encryption::AesDecryptor* aes_decryptor_; + std::string key_; + std::string file_aad_; + std::string aad_; + ::arrow::MemoryPool* pool_; +}; + +class InternalFileDecryptor { + public: + explicit InternalFileDecryptor(FileDecryptionProperties* properties, + const std::string& file_aad, + ParquetCipher::type algorithm, + const std::string& footer_key_metadata, + ::arrow::MemoryPool* pool); + + std::string& file_aad() { return file_aad_; } + + std::string GetFooterKey(); + + ParquetCipher::type algorithm() { return algorithm_; } + + std::string& footer_key_metadata() { return footer_key_metadata_; } + + FileDecryptionProperties* properties() { return properties_; } + + void WipeOutDecryptionKeys(); + + ::arrow::MemoryPool* pool() { return pool_; } + + std::shared_ptr GetFooterDecryptor(); + std::shared_ptr GetFooterDecryptorForColumnMeta(const std::string& aad = ""); + std::shared_ptr GetFooterDecryptorForColumnData(const std::string& aad = ""); + std::shared_ptr GetColumnMetaDecryptor( + const std::string& column_path, const std::string& column_key_metadata, + const std::string& aad = ""); + std::shared_ptr GetColumnDataDecryptor( + const std::string& column_path, const std::string& column_key_metadata, + const std::string& aad = ""); + + private: + FileDecryptionProperties* properties_; + // Concatenation of aad_prefix (if exists) and aad_file_unique + std::string file_aad_; + std::map> column_data_map_; + std::map> column_metadata_map_; + + std::shared_ptr footer_metadata_decryptor_; + std::shared_ptr footer_data_decryptor_; + ParquetCipher::type algorithm_; + std::string footer_key_metadata_; + std::vector all_decryptors_; + + /// Key must be 16, 24 or 32 bytes in length. Thus there could be up to three + // types of meta_decryptors and data_decryptors. + std::unique_ptr meta_decryptor_[3]; + std::unique_ptr data_decryptor_[3]; + + ::arrow::MemoryPool* pool_; + + std::shared_ptr GetFooterDecryptor(const std::string& aad, bool metadata); + std::shared_ptr GetColumnDecryptor(const std::string& column_path, + const std::string& column_key_metadata, + const std::string& aad, + bool metadata = false); + + encryption::AesDecryptor* GetMetaAesDecryptor(size_t key_size); + encryption::AesDecryptor* GetDataAesDecryptor(size_t key_size); + + int MapKeyLenToDecryptorArrayIndex(int key_len); +}; + +} // namespace parquet diff --git a/include/internal_file_decryptor_20.h b/include/internal_file_decryptor_20.h new file mode 100644 index 00000000..cc0e315e --- /dev/null +++ b/src/s3select/include/internal_file_decryptor_20.h @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "parquet/schema.h" + +namespace parquet { + +namespace encryption { +class AesDecryptor; +class AesEncryptor; +} // namespace encryption + +class ColumnCryptoMetaData; +class FileDecryptionProperties; + +// An object handling decryption using well-known encryption parameters +// +// CAUTION: Decryptor objects are not thread-safe. +class PARQUET_EXPORT Decryptor { + public: + Decryptor(std::unique_ptr decryptor, const std::string& key, + const std::string& file_aad, const std::string& aad, + ::arrow::MemoryPool* pool); + ~Decryptor(); + + const std::string& file_aad() const { return file_aad_; } + void UpdateAad(const std::string& aad) { aad_ = aad; } + ::arrow::MemoryPool* pool() { return pool_; } + + [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const; + [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const; + int32_t Decrypt(::arrow::util::span ciphertext, + ::arrow::util::span plaintext); + + private: + std::unique_ptr aes_decryptor_; + std::string key_; + std::string file_aad_; + std::string aad_; + ::arrow::MemoryPool* pool_; +}; + +class InternalFileDecryptor { + public: + explicit InternalFileDecryptor(std::shared_ptr properties, + const std::string& file_aad, + ParquetCipher::type algorithm, + const std::string& footer_key_metadata, + ::arrow::MemoryPool* pool); + + const std::string& file_aad() const { return file_aad_; } + + std::string GetFooterKey(); + + ParquetCipher::type algorithm() const { return algorithm_; } + + const std::string& footer_key_metadata() const { return footer_key_metadata_; } + + const std::shared_ptr& properties() const { + return properties_; + } + + ::arrow::MemoryPool* pool() const { return pool_; } + + // Get a Decryptor instance for the Parquet footer + std::unique_ptr GetFooterDecryptor(); + + // Get a Decryptor instance for column chunk metadata. + std::unique_ptr GetColumnMetaDecryptor( + const std::string& column_path, const std::string& column_key_metadata, + const std::string& aad = "") { + return GetColumnDecryptor(column_path, column_key_metadata, aad, /*metadata=*/true); + } + + // Get a Decryptor instance for column chunk data. + std::unique_ptr GetColumnDataDecryptor( + const std::string& column_path, const std::string& column_key_metadata, + const std::string& aad = "") { + return GetColumnDecryptor(column_path, column_key_metadata, aad, /*metadata=*/false); + } + + // Get a Decryptor factory for column chunk metadata. + // + // This is typically useful if multi-threaded decryption is expected. + // This is a static function as it accepts a null `InternalFileDecryptor*` + // argument if the column is not encrypted. + static std::function()> GetColumnMetaDecryptorFactory( + InternalFileDecryptor*, const ColumnCryptoMetaData* crypto_metadata, + const std::string& aad = ""); + // Get a Decryptor factory for column chunk data. + // + // This is typically useful if multi-threaded decryption is expected. + // This is a static function as it accepts a null `InternalFileDecryptor*` + // argument if the column is not encrypted. + static std::function()> GetColumnDataDecryptorFactory( + InternalFileDecryptor*, const ColumnCryptoMetaData* crypto_metadata, + const std::string& aad = ""); + + private: + std::shared_ptr properties_; + // Concatenation of aad_prefix (if exists) and aad_file_unique + std::string file_aad_; + ParquetCipher::type algorithm_; + std::string footer_key_metadata_; + ::arrow::MemoryPool* pool_; + + // Protects footer_key_ updates + std::mutex mutex_; + std::string footer_key_; + + std::string GetColumnKey(const std::string& column_path, + const std::string& column_key_metadata); + + std::unique_ptr GetFooterDecryptor(const std::string& aad, bool metadata); + + std::unique_ptr GetColumnDecryptor(const std::string& column_path, + const std::string& column_key_metadata, + const std::string& aad, bool metadata); + + std::function()> GetColumnDecryptorFactory( + const ColumnCryptoMetaData* crypto_metadata, const std::string& aad, bool metadata); +}; + +void UpdateDecryptor(Decryptor* decryptor, int16_t row_group_ordinal, + int16_t column_ordinal, int8_t module_type); + +} // namespace parquet diff --git a/include/s3select_parquet_intrf.h b/include/s3select_parquet_intrf.h index 8dad80ef..782d9e49 100644 --- a/src/s3select/include/s3select_parquet_intrf.h +++ b/src/s3select/include/s3select_parquet_intrf.h @@ -1002,6 +1002,7 @@ class SerializedRowGroup : public RowGroupReader::Contents { throw ParquetException("Encrypted files cannot contain more than 32767 row groups"); } +#if ARROW_VERSION_MAJOR < 20 // The column is encrypted std::shared_ptr<::parquet::Decryptor> meta_decryptor; std::shared_ptr data_decryptor; @@ -1035,6 +1036,25 @@ class SerializedRowGroup : public RowGroupReader::Contents { false, #endif properties_.memory_pool(), &ctx); +#else + // Arrow 20+ version uses factory functions instead of shared_ptr for decryptors + std::function()> meta_decryptor_factory = + InternalFileDecryptor::GetColumnMetaDecryptorFactory(file_decryptor_.get(), crypto_metadata.get()); + std::function()> data_decryptor_factory = + InternalFileDecryptor::GetColumnDataDecryptorFactory(file_decryptor_.get(), crypto_metadata.get()); + + const CryptoContext ctx { + col->has_dictionary_page(), + row_group_ordinal_, + static_cast(i), + meta_decryptor_factory, + data_decryptor_factory, + }; + + return PageReader::Open(stream, col->num_values(), col->compression(), + false, + properties_.memory_pool(), &ctx); +#endif } private: @@ -1071,7 +1091,9 @@ class SerializedFile : public ParquetFileReader::Contents { } void Close() override { +#if ARROW_VERSION_MAJOR < 20 if (file_decryptor_) file_decryptor_->WipeOutDecryptionKeys(); +#endif } std::shared_ptr GetRowGroup(int i) override { @@ -1249,9 +1271,17 @@ void SerializedFile::ParseMetaDataOfEncryptedFileWithEncryptedFooter( // Handle AAD prefix EncryptionAlgorithm algo = file_crypto_metadata->encryption_algorithm(); std::string file_aad = HandleAadPrefix(file_decryption_properties, algo); +#if ARROW_VERSION_MAJOR < 20 file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>( file_decryption_properties, file_aad, algo.algorithm, file_crypto_metadata->key_metadata(), properties_.memory_pool()); +#else + // Arrow 20+ takes a shared_ptr to FileDecryptionProperties + file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>( + std::shared_ptr(file_decryption_properties), + file_aad, algo.algorithm, + file_crypto_metadata->key_metadata(), properties_.memory_pool()); +#endif int64_t metadata_offset = source_size_ - kFooterSize - footer_len + crypto_metadata_len; uint32_t metadata_len = footer_len - crypto_metadata_len; @@ -1282,9 +1312,18 @@ void SerializedFile::ParseMetaDataOfEncryptedFileWithPlaintextFooter( EncryptionAlgorithm algo = file_metadata_->encryption_algorithm(); // Handle AAD prefix std::string file_aad = HandleAadPrefix(file_decryption_properties, algo); +#if ARROW_VERSION_MAJOR < 20 file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>( file_decryption_properties, file_aad, algo.algorithm, file_metadata_->footer_signing_key_metadata(), properties_.memory_pool()); +#else + // Arrow 20+ takes a shared_ptr to FileDecryptionProperties + file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>( + std::shared_ptr(file_decryption_properties), + file_aad, algo.algorithm, + file_metadata_->footer_signing_key_metadata(), properties_.memory_pool()); + // In Arrow 20+, no need to set file_decryptor in metadata +#endif // set the InternalFileDecryptor in the metadata as well, as it's used // for signature verification and for ColumnChunkMetaData creation. #if GAL_set_file_decryptor_declare_private