projects/aports·blob master · refs

879 lines · 33.7 KiB

1https://github.com/ceph/s3select/pull/169
2
3From 8da50fe3138e72831efd935d0d41c329974ce760 Mon Sep 17 00:00:00 2001
4From: James Page <[email protected]>
5Date: Tue, 6 May 2025 14:59:48 +0100
6Subject: [PATCH] Refactor crypto integration with arrow
7
8arrow >= 20.0.0 refactored the crypto API to support better
9multithreading; align to new API with backwards compatibility.
10---
11 include/encryption_internal.h | 100 +-----------------
12 include/encryption_internal_19.h | 114 +++++++++++++++++++++
13 include/encryption_internal_20.h | 141 +++++++++++++++++++++++++
14 include/internal_file_decryptor.h | 107 +------------------
15 include/internal_file_decryptor_19.h | 121 ++++++++++++++++++++++
16 include/internal_file_decryptor_20.h | 148 +++++++++++++++++++++++++++
17 include/s3select_parquet_intrf.h | 39 +++++++
18 7 files changed, 573 insertions(+), 197 deletions(-)
19 create mode 100644 include/encryption_internal_19.h
20 create mode 100644 include/encryption_internal_20.h
21 create mode 100644 include/internal_file_decryptor_19.h
22 create mode 100644 include/internal_file_decryptor_20.h
23
24diff --git a/include/encryption_internal.h b/include/encryption_internal.h
25index c8deee49..00129a2e 100644
26--- a/src/s3select/include/encryption_internal.h
27+++ b/src/s3select/include/encryption_internal.h
28@@ -17,98 +17,8 @@
29
30 #pragma once
31
32-#include <memory>
33-#include <string>
34-#include <vector>
35-
36-#include "parquet/properties.h"
37-#include "parquet/types.h"
38-
39-using parquet::ParquetCipher;
40-
41-namespace parquet {
42-namespace encryption {
43-
44-constexpr int kGcmTagLength = 16;
45-constexpr int kNonceLength = 12;
46-
47-// Module types
48-constexpr int8_t kFooter = 0;
49-constexpr int8_t kColumnMetaData = 1;
50-constexpr int8_t kDataPage = 2;
51-constexpr int8_t kDictionaryPage = 3;
52-constexpr int8_t kDataPageHeader = 4;
53-constexpr int8_t kDictionaryPageHeader = 5;
54-constexpr int8_t kColumnIndex = 6;
55-constexpr int8_t kOffsetIndex = 7;
56-
57-/// Performs AES encryption operations with GCM or CTR ciphers.
58-class AesEncryptor {
59- public:
60- static AesEncryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
61- std::vector<AesEncryptor*>* all_encryptors);
62-
63- ~AesEncryptor();
64-
65- /// Size difference between plaintext and ciphertext, for this cipher.
66- int CiphertextSizeDelta();
67-
68- /// Encrypts plaintext with the key and aad. Key length is passed only for validation.
69- /// If different from value in constructor, exception will be thrown.
70- int Encrypt(const uint8_t* plaintext, int plaintext_len, const uint8_t* key,
71- int key_len, const uint8_t* aad, int aad_len, uint8_t* ciphertext);
72-
73- /// Encrypts plaintext footer, in order to compute footer signature (tag).
74- int SignedFooterEncrypt(const uint8_t* footer, int footer_len, const uint8_t* key,
75- int key_len, const uint8_t* aad, int aad_len,
76- const uint8_t* nonce, uint8_t* encrypted_footer);
77-
78- void WipeOut();
79-
80- private:
81- /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
82- explicit AesEncryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
83- // PIMPL Idiom
84- class AesEncryptorImpl;
85- std::unique_ptr<AesEncryptorImpl> impl_;
86-};
87-
88-/// Performs AES decryption operations with GCM or CTR ciphers.
89-class AesDecryptor {
90- public:
91- static AesDecryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
92- std::vector<AesDecryptor*>* all_decryptors);
93-
94- ~AesDecryptor();
95- void WipeOut();
96-
97- /// Size difference between plaintext and ciphertext, for this cipher.
98- int CiphertextSizeDelta();
99-
100- /// Decrypts ciphertext with the key and aad. Key length is passed only for
101- /// validation. If different from value in constructor, exception will be thrown.
102- int Decrypt(const uint8_t* ciphertext, int ciphertext_len, const uint8_t* key,
103- int key_len, const uint8_t* aad, int aad_len, uint8_t* plaintext);
104-
105- private:
106- /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
107- explicit AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
108- // PIMPL Idiom
109- class AesDecryptorImpl;
110- std::unique_ptr<AesDecryptorImpl> impl_;
111-};
112-
113-std::string CreateModuleAad(const std::string& file_aad, int8_t module_type,
114- int16_t row_group_ordinal, int16_t column_ordinal,
115- int16_t page_ordinal);
116-
117-std::string CreateFooterAad(const std::string& aad_prefix_bytes);
118-
119-// Update last two bytes of page (or page header) module AAD
120-void QuickUpdatePageAad(const std::string& AAD, int16_t new_page_ordinal);
121-
122-// Wraps OpenSSL RAND_bytes function
123-void RandBytes(unsigned char* buf, int num);
124-
125-} // namespace encryption
126-} // namespace parquet
127+#if ARROW_VERSION_MAJOR < 20
128+#include "encryption_internal_19.h"
129+#else
130+#include "encryption_internal_20.h"
131+#endif
132diff --git a/include/encryption_internal_19.h b/include/encryption_internal_19.h
133new file mode 100644
134index 00000000..c8deee49
135--- /dev/null
136+++ b/src/s3select/include/encryption_internal_19.h
137@@ -0,0 +1,114 @@
138+// Licensed to the Apache Software Foundation (ASF) under one
139+// or more contributor license agreements. See the NOTICE file
140+// distributed with this work for additional information
141+// regarding copyright ownership. The ASF licenses this file
142+// to you under the Apache License, Version 2.0 (the
143+// "License"); you may not use this file except in compliance
144+// with the License. You may obtain a copy of the License at
145+//
146+// http://www.apache.org/licenses/LICENSE-2.0
147+//
148+// Unless required by applicable law or agreed to in writing,
149+// software distributed under the License is distributed on an
150+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
151+// KIND, either express or implied. See the License for the
152+// specific language governing permissions and limitations
153+// under the License.
154+
155+#pragma once
156+
157+#include <memory>
158+#include <string>
159+#include <vector>
160+
161+#include "parquet/properties.h"
162+#include "parquet/types.h"
163+
164+using parquet::ParquetCipher;
165+
166+namespace parquet {
167+namespace encryption {
168+
169+constexpr int kGcmTagLength = 16;
170+constexpr int kNonceLength = 12;
171+
172+// Module types
173+constexpr int8_t kFooter = 0;
174+constexpr int8_t kColumnMetaData = 1;
175+constexpr int8_t kDataPage = 2;
176+constexpr int8_t kDictionaryPage = 3;
177+constexpr int8_t kDataPageHeader = 4;
178+constexpr int8_t kDictionaryPageHeader = 5;
179+constexpr int8_t kColumnIndex = 6;
180+constexpr int8_t kOffsetIndex = 7;
181+
182+/// Performs AES encryption operations with GCM or CTR ciphers.
183+class AesEncryptor {
184+ public:
185+ static AesEncryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
186+ std::vector<AesEncryptor*>* all_encryptors);
187+
188+ ~AesEncryptor();
189+
190+ /// Size difference between plaintext and ciphertext, for this cipher.
191+ int CiphertextSizeDelta();
192+
193+ /// Encrypts plaintext with the key and aad. Key length is passed only for validation.
194+ /// If different from value in constructor, exception will be thrown.
195+ int Encrypt(const uint8_t* plaintext, int plaintext_len, const uint8_t* key,
196+ int key_len, const uint8_t* aad, int aad_len, uint8_t* ciphertext);
197+
198+ /// Encrypts plaintext footer, in order to compute footer signature (tag).
199+ int SignedFooterEncrypt(const uint8_t* footer, int footer_len, const uint8_t* key,
200+ int key_len, const uint8_t* aad, int aad_len,
201+ const uint8_t* nonce, uint8_t* encrypted_footer);
202+
203+ void WipeOut();
204+
205+ private:
206+ /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
207+ explicit AesEncryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
208+ // PIMPL Idiom
209+ class AesEncryptorImpl;
210+ std::unique_ptr<AesEncryptorImpl> impl_;
211+};
212+
213+/// Performs AES decryption operations with GCM or CTR ciphers.
214+class AesDecryptor {
215+ public:
216+ static AesDecryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
217+ std::vector<AesDecryptor*>* all_decryptors);
218+
219+ ~AesDecryptor();
220+ void WipeOut();
221+
222+ /// Size difference between plaintext and ciphertext, for this cipher.
223+ int CiphertextSizeDelta();
224+
225+ /// Decrypts ciphertext with the key and aad. Key length is passed only for
226+ /// validation. If different from value in constructor, exception will be thrown.
227+ int Decrypt(const uint8_t* ciphertext, int ciphertext_len, const uint8_t* key,
228+ int key_len, const uint8_t* aad, int aad_len, uint8_t* plaintext);
229+
230+ private:
231+ /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
232+ explicit AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
233+ // PIMPL Idiom
234+ class AesDecryptorImpl;
235+ std::unique_ptr<AesDecryptorImpl> impl_;
236+};
237+
238+std::string CreateModuleAad(const std::string& file_aad, int8_t module_type,
239+ int16_t row_group_ordinal, int16_t column_ordinal,
240+ int16_t page_ordinal);
241+
242+std::string CreateFooterAad(const std::string& aad_prefix_bytes);
243+
244+// Update last two bytes of page (or page header) module AAD
245+void QuickUpdatePageAad(const std::string& AAD, int16_t new_page_ordinal);
246+
247+// Wraps OpenSSL RAND_bytes function
248+void RandBytes(unsigned char* buf, int num);
249+
250+} // namespace encryption
251+} // namespace parquet
252diff --git a/include/encryption_internal_20.h b/include/encryption_internal_20.h
253new file mode 100644
254index 00000000..06252749
255--- /dev/null
256+++ b/src/s3select/include/encryption_internal_20.h
257@@ -0,0 +1,141 @@
258+// Licensed to the Apache Software Foundation (ASF) under one
259+// or more contributor license agreements. See the NOTICE file
260+// distributed with this work for additional information
261+// regarding copyright ownership. The ASF licenses this file
262+// to you under the Apache License, Version 2.0 (the
263+// "License"); you may not use this file except in compliance
264+// with the License. You may obtain a copy of the License at
265+//
266+// http://www.apache.org/licenses/LICENSE-2.0
267+//
268+// Unless required by applicable law or agreed to in writing,
269+// software distributed under the License is distributed on an
270+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
271+// KIND, either express or implied. See the License for the
272+// specific language governing permissions and limitations
273+// under the License.
274+
275+#pragma once
276+
277+#include <memory>
278+#include <string>
279+#include <vector>
280+
281+#include "arrow/util/span.h"
282+#include "parquet/properties.h"
283+#include "parquet/types.h"
284+
285+using parquet::ParquetCipher;
286+
287+namespace parquet::encryption {
288+
289+constexpr int32_t kGcmTagLength = 16;
290+constexpr int32_t kNonceLength = 12;
291+
292+// Module types
293+constexpr int8_t kFooter = 0;
294+constexpr int8_t kColumnMetaData = 1;
295+constexpr int8_t kDataPage = 2;
296+constexpr int8_t kDictionaryPage = 3;
297+constexpr int8_t kDataPageHeader = 4;
298+constexpr int8_t kDictionaryPageHeader = 5;
299+constexpr int8_t kColumnIndex = 6;
300+constexpr int8_t kOffsetIndex = 7;
301+constexpr int8_t kBloomFilterHeader = 8;
302+constexpr int8_t kBloomFilterBitset = 9;
303+
304+/// Performs AES encryption operations with GCM or CTR ciphers.
305+class PARQUET_EXPORT AesEncryptor {
306+ public:
307+ /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
308+ /// If write_length is true, prepend ciphertext length to the ciphertext
309+ explicit AesEncryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata,
310+ bool write_length = true);
311+
312+ static std::unique_ptr<AesEncryptor> Make(ParquetCipher::type alg_id, int32_t key_len,
313+ bool metadata, bool write_length = true);
314+
315+ ~AesEncryptor();
316+
317+ /// The size of the ciphertext, for this cipher and the specified plaintext length.
318+ [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const;
319+
320+ /// Encrypts plaintext with the key and aad. Key length is passed only for validation.
321+ /// If different from value in constructor, exception will be thrown.
322+ int32_t Encrypt(::arrow::util::span<const uint8_t> plaintext,
323+ ::arrow::util::span<const uint8_t> key,
324+ ::arrow::util::span<const uint8_t> aad,
325+ ::arrow::util::span<uint8_t> ciphertext);
326+
327+ /// Encrypts plaintext footer, in order to compute footer signature (tag).
328+ int32_t SignedFooterEncrypt(::arrow::util::span<const uint8_t> footer,
329+ ::arrow::util::span<const uint8_t> key,
330+ ::arrow::util::span<const uint8_t> aad,
331+ ::arrow::util::span<const uint8_t> nonce,
332+ ::arrow::util::span<uint8_t> encrypted_footer);
333+
334+ private:
335+ // PIMPL Idiom
336+ class AesEncryptorImpl;
337+ std::unique_ptr<AesEncryptorImpl> impl_;
338+};
339+
340+/// Performs AES decryption operations with GCM or CTR ciphers.
341+class PARQUET_EXPORT AesDecryptor {
342+ public:
343+ /// \brief Construct an AesDecryptor
344+ ///
345+ /// \param alg_id the encryption algorithm to use
346+ /// \param key_len key length. Possible values: 16, 24, 32 bytes.
347+ /// \param metadata if true then this is a metadata decryptor
348+ /// \param contains_length if true, expect ciphertext length prepended to the ciphertext
349+ explicit AesDecryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata,
350+ bool contains_length = true);
351+
352+ static std::unique_ptr<AesDecryptor> Make(ParquetCipher::type alg_id, int32_t key_len,
353+ bool metadata);
354+
355+ ~AesDecryptor();
356+
357+ /// The size of the plaintext, for this cipher and the specified ciphertext length.
358+ [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const;
359+
360+ /// The size of the ciphertext, for this cipher and the specified plaintext length.
361+ [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const;
362+
363+ /// Decrypts ciphertext with the key and aad. Key length is passed only for
364+ /// validation. If different from value in constructor, exception will be thrown.
365+ /// The caller is responsible for ensuring that the plaintext buffer is at least as
366+ /// large as PlaintextLength(ciphertext_len).
367+ int32_t Decrypt(::arrow::util::span<const uint8_t> ciphertext,
368+ ::arrow::util::span<const uint8_t> key,
369+ ::arrow::util::span<const uint8_t> aad,
370+ ::arrow::util::span<uint8_t> plaintext);
371+
372+ private:
373+ // PIMPL Idiom
374+ class AesDecryptorImpl;
375+ std::unique_ptr<AesDecryptorImpl> impl_;
376+};
377+
378+std::string CreateModuleAad(const std::string& file_aad, int8_t module_type,
379+ int16_t row_group_ordinal, int16_t column_ordinal,
380+ int32_t page_ordinal);
381+
382+std::string CreateFooterAad(const std::string& aad_prefix_bytes);
383+
384+// Update last two bytes of page (or page header) module AAD
385+void QuickUpdatePageAad(int32_t new_page_ordinal, std::string* AAD);
386+
387+// Wraps OpenSSL RAND_bytes function
388+void RandBytes(unsigned char* buf, size_t num);
389+
390+// Ensure OpenSSL is initialized.
391+//
392+// This is only necessary in specific situations since OpenSSL otherwise
393+// initializes itself automatically. For example, under Valgrind, a memory
394+// leak will be reported if OpenSSL is initialized for the first time from
395+// a worker thread; calling this function from the main thread prevents this.
396+void EnsureBackendInitialized();
397+
398+} // namespace parquet::encryption
399diff --git a/include/internal_file_decryptor.h b/include/internal_file_decryptor.h
400index 011c4acb..1e4f4b99 100644
401--- a/src/s3select/include/internal_file_decryptor.h
402+++ b/src/s3select/include/internal_file_decryptor.h
403@@ -17,105 +17,8 @@
404
405 #pragma once
406
407-#include <map>
408-#include <memory>
409-#include <string>
410-#include <vector>
411-
412-#include "parquet/schema.h"
413-
414-namespace parquet {
415-
416-namespace encryption {
417-class AesDecryptor;
418-class AesEncryptor;
419-} // namespace encryption
420-
421-class FileDecryptionProperties;
422-
423-class PARQUET_EXPORT Decryptor {
424- public:
425- Decryptor(encryption::AesDecryptor* decryptor, const std::string& key,
426- const std::string& file_aad, const std::string& aad,
427- ::arrow::MemoryPool* pool);
428-
429- const std::string& file_aad() const { return file_aad_; }
430- void UpdateAad(const std::string& aad) { aad_ = aad; }
431- ::arrow::MemoryPool* pool() { return pool_; }
432-
433- int CiphertextSizeDelta();
434- int Decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext);
435-
436- private:
437- encryption::AesDecryptor* aes_decryptor_;
438- std::string key_;
439- std::string file_aad_;
440- std::string aad_;
441- ::arrow::MemoryPool* pool_;
442-};
443-
444-class InternalFileDecryptor {
445- public:
446- explicit InternalFileDecryptor(FileDecryptionProperties* properties,
447- const std::string& file_aad,
448- ParquetCipher::type algorithm,
449- const std::string& footer_key_metadata,
450- ::arrow::MemoryPool* pool);
451-
452- std::string& file_aad() { return file_aad_; }
453-
454- std::string GetFooterKey();
455-
456- ParquetCipher::type algorithm() { return algorithm_; }
457-
458- std::string& footer_key_metadata() { return footer_key_metadata_; }
459-
460- FileDecryptionProperties* properties() { return properties_; }
461-
462- void WipeOutDecryptionKeys();
463-
464- ::arrow::MemoryPool* pool() { return pool_; }
465-
466- std::shared_ptr<Decryptor> GetFooterDecryptor();
467- std::shared_ptr<Decryptor> GetFooterDecryptorForColumnMeta(const std::string& aad = "");
468- std::shared_ptr<Decryptor> GetFooterDecryptorForColumnData(const std::string& aad = "");
469- std::shared_ptr<Decryptor> GetColumnMetaDecryptor(
470- const std::string& column_path, const std::string& column_key_metadata,
471- const std::string& aad = "");
472- std::shared_ptr<Decryptor> GetColumnDataDecryptor(
473- const std::string& column_path, const std::string& column_key_metadata,
474- const std::string& aad = "");
475-
476- private:
477- FileDecryptionProperties* properties_;
478- // Concatenation of aad_prefix (if exists) and aad_file_unique
479- std::string file_aad_;
480- std::map<std::string, std::shared_ptr<Decryptor>> column_data_map_;
481- std::map<std::string, std::shared_ptr<Decryptor>> column_metadata_map_;
482-
483- std::shared_ptr<Decryptor> footer_metadata_decryptor_;
484- std::shared_ptr<Decryptor> footer_data_decryptor_;
485- ParquetCipher::type algorithm_;
486- std::string footer_key_metadata_;
487- std::vector<encryption::AesDecryptor*> all_decryptors_;
488-
489- /// Key must be 16, 24 or 32 bytes in length. Thus there could be up to three
490- // types of meta_decryptors and data_decryptors.
491- std::unique_ptr<encryption::AesDecryptor> meta_decryptor_[3];
492- std::unique_ptr<encryption::AesDecryptor> data_decryptor_[3];
493-
494- ::arrow::MemoryPool* pool_;
495-
496- std::shared_ptr<Decryptor> GetFooterDecryptor(const std::string& aad, bool metadata);
497- std::shared_ptr<Decryptor> GetColumnDecryptor(const std::string& column_path,
498- const std::string& column_key_metadata,
499- const std::string& aad,
500- bool metadata = false);
501-
502- encryption::AesDecryptor* GetMetaAesDecryptor(size_t key_size);
503- encryption::AesDecryptor* GetDataAesDecryptor(size_t key_size);
504-
505- int MapKeyLenToDecryptorArrayIndex(int key_len);
506-};
507-
508-} // namespace parquet
509+#if ARROW_VERSION_MAJOR < 20
510+#include "internal_file_decryptor_19.h"
511+#else
512+#include "internal_file_decryptor_20.h"
513+#endif
514diff --git a/include/internal_file_decryptor_19.h b/include/internal_file_decryptor_19.h
515new file mode 100644
516index 00000000..011c4acb
517--- /dev/null
518+++ b/src/s3select/include/internal_file_decryptor_19.h
519@@ -0,0 +1,121 @@
520+// Licensed to the Apache Software Foundation (ASF) under one
521+// or more contributor license agreements. See the NOTICE file
522+// distributed with this work for additional information
523+// regarding copyright ownership. The ASF licenses this file
524+// to you under the Apache License, Version 2.0 (the
525+// "License"); you may not use this file except in compliance
526+// with the License. You may obtain a copy of the License at
527+//
528+// http://www.apache.org/licenses/LICENSE-2.0
529+//
530+// Unless required by applicable law or agreed to in writing,
531+// software distributed under the License is distributed on an
532+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
533+// KIND, either express or implied. See the License for the
534+// specific language governing permissions and limitations
535+// under the License.
536+
537+#pragma once
538+
539+#include <map>
540+#include <memory>
541+#include <string>
542+#include <vector>
543+
544+#include "parquet/schema.h"
545+
546+namespace parquet {
547+
548+namespace encryption {
549+class AesDecryptor;
550+class AesEncryptor;
551+} // namespace encryption
552+
553+class FileDecryptionProperties;
554+
555+class PARQUET_EXPORT Decryptor {
556+ public:
557+ Decryptor(encryption::AesDecryptor* decryptor, const std::string& key,
558+ const std::string& file_aad, const std::string& aad,
559+ ::arrow::MemoryPool* pool);
560+
561+ const std::string& file_aad() const { return file_aad_; }
562+ void UpdateAad(const std::string& aad) { aad_ = aad; }
563+ ::arrow::MemoryPool* pool() { return pool_; }
564+
565+ int CiphertextSizeDelta();
566+ int Decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext);
567+
568+ private:
569+ encryption::AesDecryptor* aes_decryptor_;
570+ std::string key_;
571+ std::string file_aad_;
572+ std::string aad_;
573+ ::arrow::MemoryPool* pool_;
574+};
575+
576+class InternalFileDecryptor {
577+ public:
578+ explicit InternalFileDecryptor(FileDecryptionProperties* properties,
579+ const std::string& file_aad,
580+ ParquetCipher::type algorithm,
581+ const std::string& footer_key_metadata,
582+ ::arrow::MemoryPool* pool);
583+
584+ std::string& file_aad() { return file_aad_; }
585+
586+ std::string GetFooterKey();
587+
588+ ParquetCipher::type algorithm() { return algorithm_; }
589+
590+ std::string& footer_key_metadata() { return footer_key_metadata_; }
591+
592+ FileDecryptionProperties* properties() { return properties_; }
593+
594+ void WipeOutDecryptionKeys();
595+
596+ ::arrow::MemoryPool* pool() { return pool_; }
597+
598+ std::shared_ptr<Decryptor> GetFooterDecryptor();
599+ std::shared_ptr<Decryptor> GetFooterDecryptorForColumnMeta(const std::string& aad = "");
600+ std::shared_ptr<Decryptor> GetFooterDecryptorForColumnData(const std::string& aad = "");
601+ std::shared_ptr<Decryptor> GetColumnMetaDecryptor(
602+ const std::string& column_path, const std::string& column_key_metadata,
603+ const std::string& aad = "");
604+ std::shared_ptr<Decryptor> GetColumnDataDecryptor(
605+ const std::string& column_path, const std::string& column_key_metadata,
606+ const std::string& aad = "");
607+
608+ private:
609+ FileDecryptionProperties* properties_;
610+ // Concatenation of aad_prefix (if exists) and aad_file_unique
611+ std::string file_aad_;
612+ std::map<std::string, std::shared_ptr<Decryptor>> column_data_map_;
613+ std::map<std::string, std::shared_ptr<Decryptor>> column_metadata_map_;
614+
615+ std::shared_ptr<Decryptor> footer_metadata_decryptor_;
616+ std::shared_ptr<Decryptor> footer_data_decryptor_;
617+ ParquetCipher::type algorithm_;
618+ std::string footer_key_metadata_;
619+ std::vector<encryption::AesDecryptor*> all_decryptors_;
620+
621+ /// Key must be 16, 24 or 32 bytes in length. Thus there could be up to three
622+ // types of meta_decryptors and data_decryptors.
623+ std::unique_ptr<encryption::AesDecryptor> meta_decryptor_[3];
624+ std::unique_ptr<encryption::AesDecryptor> data_decryptor_[3];
625+
626+ ::arrow::MemoryPool* pool_;
627+
628+ std::shared_ptr<Decryptor> GetFooterDecryptor(const std::string& aad, bool metadata);
629+ std::shared_ptr<Decryptor> GetColumnDecryptor(const std::string& column_path,
630+ const std::string& column_key_metadata,
631+ const std::string& aad,
632+ bool metadata = false);
633+
634+ encryption::AesDecryptor* GetMetaAesDecryptor(size_t key_size);
635+ encryption::AesDecryptor* GetDataAesDecryptor(size_t key_size);
636+
637+ int MapKeyLenToDecryptorArrayIndex(int key_len);
638+};
639+
640+} // namespace parquet
641diff --git a/include/internal_file_decryptor_20.h b/include/internal_file_decryptor_20.h
642new file mode 100644
643index 00000000..cc0e315e
644--- /dev/null
645+++ b/src/s3select/include/internal_file_decryptor_20.h
646@@ -0,0 +1,148 @@
647+// Licensed to the Apache Software Foundation (ASF) under one
648+// or more contributor license agreements. See the NOTICE file
649+// distributed with this work for additional information
650+// regarding copyright ownership. The ASF licenses this file
651+// to you under the Apache License, Version 2.0 (the
652+// "License"); you may not use this file except in compliance
653+// with the License. You may obtain a copy of the License at
654+//
655+// http://www.apache.org/licenses/LICENSE-2.0
656+//
657+// Unless required by applicable law or agreed to in writing,
658+// software distributed under the License is distributed on an
659+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
660+// KIND, either express or implied. See the License for the
661+// specific language governing permissions and limitations
662+// under the License.
663+
664+#pragma once
665+
666+#include <memory>
667+#include <mutex>
668+#include <string>
669+#include <vector>
670+
671+#include "parquet/schema.h"
672+
673+namespace parquet {
674+
675+namespace encryption {
676+class AesDecryptor;
677+class AesEncryptor;
678+} // namespace encryption
679+
680+class ColumnCryptoMetaData;
681+class FileDecryptionProperties;
682+
683+// An object handling decryption using well-known encryption parameters
684+//
685+// CAUTION: Decryptor objects are not thread-safe.
686+class PARQUET_EXPORT Decryptor {
687+ public:
688+ Decryptor(std::unique_ptr<encryption::AesDecryptor> decryptor, const std::string& key,
689+ const std::string& file_aad, const std::string& aad,
690+ ::arrow::MemoryPool* pool);
691+ ~Decryptor();
692+
693+ const std::string& file_aad() const { return file_aad_; }
694+ void UpdateAad(const std::string& aad) { aad_ = aad; }
695+ ::arrow::MemoryPool* pool() { return pool_; }
696+
697+ [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const;
698+ [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const;
699+ int32_t Decrypt(::arrow::util::span<const uint8_t> ciphertext,
700+ ::arrow::util::span<uint8_t> plaintext);
701+
702+ private:
703+ std::unique_ptr<encryption::AesDecryptor> aes_decryptor_;
704+ std::string key_;
705+ std::string file_aad_;
706+ std::string aad_;
707+ ::arrow::MemoryPool* pool_;
708+};
709+
710+class InternalFileDecryptor {
711+ public:
712+ explicit InternalFileDecryptor(std::shared_ptr<FileDecryptionProperties> properties,
713+ const std::string& file_aad,
714+ ParquetCipher::type algorithm,
715+ const std::string& footer_key_metadata,
716+ ::arrow::MemoryPool* pool);
717+
718+ const std::string& file_aad() const { return file_aad_; }
719+
720+ std::string GetFooterKey();
721+
722+ ParquetCipher::type algorithm() const { return algorithm_; }
723+
724+ const std::string& footer_key_metadata() const { return footer_key_metadata_; }
725+
726+ const std::shared_ptr<FileDecryptionProperties>& properties() const {
727+ return properties_;
728+ }
729+
730+ ::arrow::MemoryPool* pool() const { return pool_; }
731+
732+ // Get a Decryptor instance for the Parquet footer
733+ std::unique_ptr<Decryptor> GetFooterDecryptor();
734+
735+ // Get a Decryptor instance for column chunk metadata.
736+ std::unique_ptr<Decryptor> GetColumnMetaDecryptor(
737+ const std::string& column_path, const std::string& column_key_metadata,
738+ const std::string& aad = "") {
739+ return GetColumnDecryptor(column_path, column_key_metadata, aad, /*metadata=*/true);
740+ }
741+
742+ // Get a Decryptor instance for column chunk data.
743+ std::unique_ptr<Decryptor> GetColumnDataDecryptor(
744+ const std::string& column_path, const std::string& column_key_metadata,
745+ const std::string& aad = "") {
746+ return GetColumnDecryptor(column_path, column_key_metadata, aad, /*metadata=*/false);
747+ }
748+
749+ // Get a Decryptor factory for column chunk metadata.
750+ //
751+ // This is typically useful if multi-threaded decryption is expected.
752+ // This is a static function as it accepts a null `InternalFileDecryptor*`
753+ // argument if the column is not encrypted.
754+ static std::function<std::unique_ptr<Decryptor>()> GetColumnMetaDecryptorFactory(
755+ InternalFileDecryptor*, const ColumnCryptoMetaData* crypto_metadata,
756+ const std::string& aad = "");
757+ // Get a Decryptor factory for column chunk data.
758+ //
759+ // This is typically useful if multi-threaded decryption is expected.
760+ // This is a static function as it accepts a null `InternalFileDecryptor*`
761+ // argument if the column is not encrypted.
762+ static std::function<std::unique_ptr<Decryptor>()> GetColumnDataDecryptorFactory(
763+ InternalFileDecryptor*, const ColumnCryptoMetaData* crypto_metadata,
764+ const std::string& aad = "");
765+
766+ private:
767+ std::shared_ptr<FileDecryptionProperties> properties_;
768+ // Concatenation of aad_prefix (if exists) and aad_file_unique
769+ std::string file_aad_;
770+ ParquetCipher::type algorithm_;
771+ std::string footer_key_metadata_;
772+ ::arrow::MemoryPool* pool_;
773+
774+ // Protects footer_key_ updates
775+ std::mutex mutex_;
776+ std::string footer_key_;
777+
778+ std::string GetColumnKey(const std::string& column_path,
779+ const std::string& column_key_metadata);
780+
781+ std::unique_ptr<Decryptor> GetFooterDecryptor(const std::string& aad, bool metadata);
782+
783+ std::unique_ptr<Decryptor> GetColumnDecryptor(const std::string& column_path,
784+ const std::string& column_key_metadata,
785+ const std::string& aad, bool metadata);
786+
787+ std::function<std::unique_ptr<Decryptor>()> GetColumnDecryptorFactory(
788+ const ColumnCryptoMetaData* crypto_metadata, const std::string& aad, bool metadata);
789+};
790+
791+void UpdateDecryptor(Decryptor* decryptor, int16_t row_group_ordinal,
792+ int16_t column_ordinal, int8_t module_type);
793+
794+} // namespace parquet
795diff --git a/include/s3select_parquet_intrf.h b/include/s3select_parquet_intrf.h
796index 8dad80ef..782d9e49 100644
797--- a/src/s3select/include/s3select_parquet_intrf.h
798+++ b/src/s3select/include/s3select_parquet_intrf.h
799@@ -1002,6 +1002,7 @@ class SerializedRowGroup : public RowGroupReader::Contents {
800 throw ParquetException("Encrypted files cannot contain more than 32767 row groups");
801 }
802
803+#if ARROW_VERSION_MAJOR < 20
804 // The column is encrypted
805 std::shared_ptr<::parquet::Decryptor> meta_decryptor;
806 std::shared_ptr<Decryptor> data_decryptor;
807@@ -1035,6 +1036,25 @@ class SerializedRowGroup : public RowGroupReader::Contents {
808 false,
809 #endif
810 properties_.memory_pool(), &ctx);
811+#else
812+ // Arrow 20+ version uses factory functions instead of shared_ptr for decryptors
813+ std::function<std::unique_ptr<Decryptor>()> meta_decryptor_factory =
814+ InternalFileDecryptor::GetColumnMetaDecryptorFactory(file_decryptor_.get(), crypto_metadata.get());
815+ std::function<std::unique_ptr<Decryptor>()> data_decryptor_factory =
816+ InternalFileDecryptor::GetColumnDataDecryptorFactory(file_decryptor_.get(), crypto_metadata.get());
817+
818+ const CryptoContext ctx {
819+ col->has_dictionary_page(),
820+ row_group_ordinal_,
821+ static_cast<int16_t>(i),
822+ meta_decryptor_factory,
823+ data_decryptor_factory,
824+ };
825+
826+ return PageReader::Open(stream, col->num_values(), col->compression(),
827+ false,
828+ properties_.memory_pool(), &ctx);
829+#endif
830 }
831
832 private:
833@@ -1071,7 +1091,9 @@ class SerializedFile : public ParquetFileReader::Contents {
834 }
835
836 void Close() override {
837+#if ARROW_VERSION_MAJOR < 20
838 if (file_decryptor_) file_decryptor_->WipeOutDecryptionKeys();
839+#endif
840 }
841
842 std::shared_ptr<RowGroupReader> GetRowGroup(int i) override {
843@@ -1249,9 +1271,17 @@ void SerializedFile::ParseMetaDataOfEncryptedFileWithEncryptedFooter(
844 // Handle AAD prefix
845 EncryptionAlgorithm algo = file_crypto_metadata->encryption_algorithm();
846 std::string file_aad = HandleAadPrefix(file_decryption_properties, algo);
847+#if ARROW_VERSION_MAJOR < 20
848 file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
849 file_decryption_properties, file_aad, algo.algorithm,
850 file_crypto_metadata->key_metadata(), properties_.memory_pool());
851+#else
852+ // Arrow 20+ takes a shared_ptr to FileDecryptionProperties
853+ file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
854+ std::shared_ptr<FileDecryptionProperties>(file_decryption_properties),
855+ file_aad, algo.algorithm,
856+ file_crypto_metadata->key_metadata(), properties_.memory_pool());
857+#endif
858
859 int64_t metadata_offset = source_size_ - kFooterSize - footer_len + crypto_metadata_len;
860 uint32_t metadata_len = footer_len - crypto_metadata_len;
861@@ -1282,9 +1312,18 @@ void SerializedFile::ParseMetaDataOfEncryptedFileWithPlaintextFooter(
862 EncryptionAlgorithm algo = file_metadata_->encryption_algorithm();
863 // Handle AAD prefix
864 std::string file_aad = HandleAadPrefix(file_decryption_properties, algo);
865+#if ARROW_VERSION_MAJOR < 20
866 file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
867 file_decryption_properties, file_aad, algo.algorithm,
868 file_metadata_->footer_signing_key_metadata(), properties_.memory_pool());
869+#else
870+ // Arrow 20+ takes a shared_ptr to FileDecryptionProperties
871+ file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
872+ std::shared_ptr<FileDecryptionProperties>(file_decryption_properties),
873+ file_aad, algo.algorithm,
874+ file_metadata_->footer_signing_key_metadata(), properties_.memory_pool());
875+ // In Arrow 20+, no need to set file_decryptor in metadata
876+#endif
877 // set the InternalFileDecryptor in the metadata as well, as it's used
878 // for signature verification and for ColumnChunkMetaData creation.
879 #if GAL_set_file_decryptor_declare_private