projects/linux·blob assoc-array-fixes · refs

1258 lines · 33.5 KiB

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3#[cfg(wrap_proc_macro)]
4use crate::imp;
5#[cfg(span_locations)]
6use crate::location::LineColumn;
7use crate::parse::{self, Cursor};
8use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut};
9use crate::{Delimiter, Spacing, TokenTree};
10#[cfg(all(span_locations, not(fuzzing)))]
11use alloc::collections::BTreeMap;
12#[cfg(all(span_locations, not(fuzzing)))]
13use core::cell::RefCell;
14#[cfg(span_locations)]
15use core::cmp;
16#[cfg(all(span_locations, not(fuzzing)))]
17use core::cmp::Ordering;
18use core::fmt::{self, Debug, Display, Write};
19use core::mem::ManuallyDrop;
20#[cfg(span_locations)]
21use core::ops::Range;
22use core::ops::RangeBounds;
23use core::ptr;
24use core::str;
25#[cfg(feature = "proc-macro")]
26use core::str::FromStr;
27use std::ffi::CStr;
28#[cfg(wrap_proc_macro)]
29use std::panic;
30#[cfg(span_locations)]
31use std::path::PathBuf;
32
33/// Force use of proc-macro2's fallback implementation of the API for now, even
34/// if the compiler's implementation is available.
35pub fn force() {
36 #[cfg(wrap_proc_macro)]
37 crate::detection::force_fallback();
38}
39
40/// Resume using the compiler's implementation of the proc macro API if it is
41/// available.
42pub fn unforce() {
43 #[cfg(wrap_proc_macro)]
44 crate::detection::unforce_fallback();
45}
46
47#[derive(Clone)]
48pub(crate) struct TokenStream {
49 inner: RcVec<TokenTree>,
50}
51
52#[derive(Debug)]
53pub(crate) struct LexError {
54 pub(crate) span: Span,
55}
56
57impl LexError {
58 pub(crate) fn span(&self) -> Span {
59 self.span
60 }
61
62 pub(crate) fn call_site() -> Self {
63 LexError {
64 span: Span::call_site(),
65 }
66 }
67}
68
69impl TokenStream {
70 pub(crate) fn new() -> Self {
71 TokenStream {
72 inner: RcVecBuilder::new().build(),
73 }
74 }
75
76 pub(crate) fn from_str_checked(src: &str) -> Result<Self, LexError> {
77 // Create a dummy file & add it to the source map
78 let mut cursor = get_cursor(src);
79
80 // Strip a byte order mark if present
81 const BYTE_ORDER_MARK: &str = "\u{feff}";
82 if cursor.starts_with(BYTE_ORDER_MARK) {
83 cursor = cursor.advance(BYTE_ORDER_MARK.len());
84 }
85
86 parse::token_stream(cursor)
87 }
88
89 #[cfg(feature = "proc-macro")]
90 pub(crate) fn from_str_unchecked(src: &str) -> Self {
91 Self::from_str_checked(src).unwrap()
92 }
93
94 pub(crate) fn is_empty(&self) -> bool {
95 self.inner.len() == 0
96 }
97
98 fn take_inner(self) -> RcVecBuilder<TokenTree> {
99 let nodrop = ManuallyDrop::new(self);
100 unsafe { ptr::read(&nodrop.inner) }.make_owned()
101 }
102}
103
104fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
105 // https://github.com/dtolnay/proc-macro2/issues/235
106 match token {
107 TokenTree::Literal(crate::Literal {
108 #[cfg(wrap_proc_macro)]
109 inner: crate::imp::Literal::Fallback(literal),
110 #[cfg(not(wrap_proc_macro))]
111 inner: literal,
112 ..
113 }) if literal.repr.starts_with('-') => {
114 push_negative_literal(vec, literal);
115 }
116 _ => vec.push(token),
117 }
118
119 #[cold]
120 fn push_negative_literal(mut vec: RcVecMut<TokenTree>, mut literal: Literal) {
121 literal.repr.remove(0);
122 let mut punct = crate::Punct::new('-', Spacing::Alone);
123 punct.set_span(crate::Span::_new_fallback(literal.span));
124 vec.push(TokenTree::Punct(punct));
125 vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal)));
126 }
127}
128
129// Nonrecursive to prevent stack overflow.
130impl Drop for TokenStream {
131 fn drop(&mut self) {
132 let mut stack = Vec::new();
133 let mut current = match self.inner.get_mut() {
134 Some(inner) => inner.take().into_iter(),
135 None => return,
136 };
137 loop {
138 while let Some(token) = current.next() {
139 let group = match token {
140 TokenTree::Group(group) => group.inner,
141 _ => continue,
142 };
143 #[cfg(wrap_proc_macro)]
144 let group = match group {
145 crate::imp::Group::Fallback(group) => group,
146 crate::imp::Group::Compiler(_) => continue,
147 };
148 let mut group = group;
149 if let Some(inner) = group.stream.inner.get_mut() {
150 stack.push(current);
151 current = inner.take().into_iter();
152 }
153 }
154 match stack.pop() {
155 Some(next) => current = next,
156 None => return,
157 }
158 }
159 }
160}
161
162pub(crate) struct TokenStreamBuilder {
163 inner: RcVecBuilder<TokenTree>,
164}
165
166impl TokenStreamBuilder {
167 pub(crate) fn new() -> Self {
168 TokenStreamBuilder {
169 inner: RcVecBuilder::new(),
170 }
171 }
172
173 pub(crate) fn with_capacity(cap: usize) -> Self {
174 TokenStreamBuilder {
175 inner: RcVecBuilder::with_capacity(cap),
176 }
177 }
178
179 pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) {
180 self.inner.push(tt);
181 }
182
183 pub(crate) fn build(self) -> TokenStream {
184 TokenStream {
185 inner: self.inner.build(),
186 }
187 }
188}
189
190#[cfg(span_locations)]
191fn get_cursor(src: &str) -> Cursor {
192 #[cfg(fuzzing)]
193 return Cursor { rest: src, off: 1 };
194
195 // Create a dummy file & add it to the source map
196 #[cfg(not(fuzzing))]
197 SOURCE_MAP.with(|sm| {
198 let mut sm = sm.borrow_mut();
199 let span = sm.add_file(src);
200 Cursor {
201 rest: src,
202 off: span.lo,
203 }
204 })
205}
206
207#[cfg(not(span_locations))]
208fn get_cursor(src: &str) -> Cursor {
209 Cursor { rest: src }
210}
211
212impl Display for LexError {
213 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
214 f.write_str("cannot parse string into token stream")
215 }
216}
217
218impl Display for TokenStream {
219 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
220 let mut joint = false;
221 for (i, tt) in self.inner.iter().enumerate() {
222 if i != 0 && !joint {
223 write!(f, " ")?;
224 }
225 joint = false;
226 match tt {
227 TokenTree::Group(tt) => Display::fmt(tt, f),
228 TokenTree::Ident(tt) => Display::fmt(tt, f),
229 TokenTree::Punct(tt) => {
230 joint = tt.spacing() == Spacing::Joint;
231 Display::fmt(tt, f)
232 }
233 TokenTree::Literal(tt) => Display::fmt(tt, f),
234 }?;
235 }
236
237 Ok(())
238 }
239}
240
241impl Debug for TokenStream {
242 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
243 f.write_str("TokenStream ")?;
244 f.debug_list().entries(self.clone()).finish()
245 }
246}
247
248#[cfg(feature = "proc-macro")]
249impl From<proc_macro::TokenStream> for TokenStream {
250 fn from(inner: proc_macro::TokenStream) -> Self {
251 TokenStream::from_str_unchecked(&inner.to_string())
252 }
253}
254
255#[cfg(feature = "proc-macro")]
256impl From<TokenStream> for proc_macro::TokenStream {
257 fn from(inner: TokenStream) -> Self {
258 proc_macro::TokenStream::from_str_unchecked(&inner.to_string())
259 }
260}
261
262impl From<TokenTree> for TokenStream {
263 fn from(tree: TokenTree) -> Self {
264 let mut stream = RcVecBuilder::new();
265 push_token_from_proc_macro(stream.as_mut(), tree);
266 TokenStream {
267 inner: stream.build(),
268 }
269 }
270}
271
272impl FromIterator<TokenTree> for TokenStream {
273 fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self {
274 let mut stream = TokenStream::new();
275 stream.extend(tokens);
276 stream
277 }
278}
279
280impl FromIterator<TokenStream> for TokenStream {
281 fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
282 let mut v = RcVecBuilder::new();
283
284 for stream in streams {
285 v.extend(stream.take_inner());
286 }
287
288 TokenStream { inner: v.build() }
289 }
290}
291
292impl Extend<TokenTree> for TokenStream {
293 fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) {
294 let mut vec = self.inner.make_mut();
295 tokens
296 .into_iter()
297 .for_each(|token| push_token_from_proc_macro(vec.as_mut(), token));
298 }
299}
300
301impl Extend<TokenStream> for TokenStream {
302 fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
303 self.inner.make_mut().extend(streams.into_iter().flatten());
304 }
305}
306
307pub(crate) type TokenTreeIter = RcVecIntoIter<TokenTree>;
308
309impl IntoIterator for TokenStream {
310 type Item = TokenTree;
311 type IntoIter = TokenTreeIter;
312
313 fn into_iter(self) -> TokenTreeIter {
314 self.take_inner().into_iter()
315 }
316}
317
318#[cfg(all(span_locations, not(fuzzing)))]
319thread_local! {
320 static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
321 // Start with a single dummy file which all call_site() and def_site()
322 // spans reference.
323 files: vec![FileInfo {
324 source_text: String::new(),
325 span: Span { lo: 0, hi: 0 },
326 lines: vec![0],
327 char_index_to_byte_offset: BTreeMap::new(),
328 }],
329 });
330}
331
332#[cfg(span_locations)]
333pub(crate) fn invalidate_current_thread_spans() {
334 #[cfg(not(fuzzing))]
335 SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1));
336}
337
338#[cfg(all(span_locations, not(fuzzing)))]
339struct FileInfo {
340 source_text: String,
341 span: Span,
342 lines: Vec<usize>,
343 char_index_to_byte_offset: BTreeMap<usize, usize>,
344}
345
346#[cfg(all(span_locations, not(fuzzing)))]
347impl FileInfo {
348 fn offset_line_column(&self, offset: usize) -> LineColumn {
349 assert!(self.span_within(Span {
350 lo: offset as u32,
351 hi: offset as u32,
352 }));
353 let offset = offset - self.span.lo as usize;
354 match self.lines.binary_search(&offset) {
355 Ok(found) => LineColumn {
356 line: found + 1,
357 column: 0,
358 },
359 Err(idx) => LineColumn {
360 line: idx,
361 column: offset - self.lines[idx - 1],
362 },
363 }
364 }
365
366 fn span_within(&self, span: Span) -> bool {
367 span.lo >= self.span.lo && span.hi <= self.span.hi
368 }
369
370 fn byte_range(&mut self, span: Span) -> Range<usize> {
371 let lo_char = (span.lo - self.span.lo) as usize;
372
373 // Look up offset of the largest already-computed char index that is
374 // less than or equal to the current requested one. We resume counting
375 // chars from that point.
376 let (&last_char_index, &last_byte_offset) = self
377 .char_index_to_byte_offset
378 .range(..=lo_char)
379 .next_back()
380 .unwrap_or((&0, &0));
381
382 let lo_byte = if last_char_index == lo_char {
383 last_byte_offset
384 } else {
385 let total_byte_offset = match self.source_text[last_byte_offset..]
386 .char_indices()
387 .nth(lo_char - last_char_index)
388 {
389 Some((additional_offset, _ch)) => last_byte_offset + additional_offset,
390 None => self.source_text.len(),
391 };
392 self.char_index_to_byte_offset
393 .insert(lo_char, total_byte_offset);
394 total_byte_offset
395 };
396
397 let trunc_lo = &self.source_text[lo_byte..];
398 let char_len = (span.hi - span.lo) as usize;
399 lo_byte..match trunc_lo.char_indices().nth(char_len) {
400 Some((offset, _ch)) => lo_byte + offset,
401 None => self.source_text.len(),
402 }
403 }
404
405 fn source_text(&mut self, span: Span) -> String {
406 let byte_range = self.byte_range(span);
407 self.source_text[byte_range].to_owned()
408 }
409}
410
411/// Computes the offsets of each line in the given source string
412/// and the total number of characters
413#[cfg(all(span_locations, not(fuzzing)))]
414fn lines_offsets(s: &str) -> (usize, Vec<usize>) {
415 let mut lines = vec![0];
416 let mut total = 0;
417
418 for ch in s.chars() {
419 total += 1;
420 if ch == '\n' {
421 lines.push(total);
422 }
423 }
424
425 (total, lines)
426}
427
428#[cfg(all(span_locations, not(fuzzing)))]
429struct SourceMap {
430 files: Vec<FileInfo>,
431}
432
433#[cfg(all(span_locations, not(fuzzing)))]
434impl SourceMap {
435 fn next_start_pos(&self) -> u32 {
436 // Add 1 so there's always space between files.
437 //
438 // We'll always have at least 1 file, as we initialize our files list
439 // with a dummy file.
440 self.files.last().unwrap().span.hi + 1
441 }
442
443 fn add_file(&mut self, src: &str) -> Span {
444 let (len, lines) = lines_offsets(src);
445 let lo = self.next_start_pos();
446 let span = Span {
447 lo,
448 hi: lo + (len as u32),
449 };
450
451 self.files.push(FileInfo {
452 source_text: src.to_owned(),
453 span,
454 lines,
455 // Populated lazily by source_text().
456 char_index_to_byte_offset: BTreeMap::new(),
457 });
458
459 span
460 }
461
462 fn find(&self, span: Span) -> usize {
463 match self.files.binary_search_by(|file| {
464 if file.span.hi < span.lo {
465 Ordering::Less
466 } else if file.span.lo > span.hi {
467 Ordering::Greater
468 } else {
469 assert!(file.span_within(span));
470 Ordering::Equal
471 }
472 }) {
473 Ok(i) => i,
474 Err(_) => unreachable!("Invalid span with no related FileInfo!"),
475 }
476 }
477
478 fn filepath(&self, span: Span) -> String {
479 let i = self.find(span);
480 if i == 0 {
481 "<unspecified>".to_owned()
482 } else {
483 format!("<parsed string {}>", i)
484 }
485 }
486
487 fn fileinfo(&self, span: Span) -> &FileInfo {
488 let i = self.find(span);
489 &self.files[i]
490 }
491
492 fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo {
493 let i = self.find(span);
494 &mut self.files[i]
495 }
496}
497
498#[derive(Clone, Copy, PartialEq, Eq)]
499pub(crate) struct Span {
500 #[cfg(span_locations)]
501 pub(crate) lo: u32,
502 #[cfg(span_locations)]
503 pub(crate) hi: u32,
504}
505
506impl Span {
507 #[cfg(not(span_locations))]
508 pub(crate) fn call_site() -> Self {
509 Span {}
510 }
511
512 #[cfg(span_locations)]
513 pub(crate) fn call_site() -> Self {
514 Span { lo: 0, hi: 0 }
515 }
516
517 pub(crate) fn mixed_site() -> Self {
518 Span::call_site()
519 }
520
521 #[cfg(procmacro2_semver_exempt)]
522 pub(crate) fn def_site() -> Self {
523 Span::call_site()
524 }
525
526 pub(crate) fn resolved_at(&self, _other: Span) -> Span {
527 // Stable spans consist only of line/column information, so
528 // `resolved_at` and `located_at` only select which span the
529 // caller wants line/column information from.
530 *self
531 }
532
533 pub(crate) fn located_at(&self, other: Span) -> Span {
534 other
535 }
536
537 #[cfg(span_locations)]
538 pub(crate) fn byte_range(&self) -> Range<usize> {
539 #[cfg(fuzzing)]
540 return 0..0;
541
542 #[cfg(not(fuzzing))]
543 {
544 if self.is_call_site() {
545 0..0
546 } else {
547 SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self))
548 }
549 }
550 }
551
552 #[cfg(span_locations)]
553 pub(crate) fn start(&self) -> LineColumn {
554 #[cfg(fuzzing)]
555 return LineColumn { line: 0, column: 0 };
556
557 #[cfg(not(fuzzing))]
558 SOURCE_MAP.with(|sm| {
559 let sm = sm.borrow();
560 let fi = sm.fileinfo(*self);
561 fi.offset_line_column(self.lo as usize)
562 })
563 }
564
565 #[cfg(span_locations)]
566 pub(crate) fn end(&self) -> LineColumn {
567 #[cfg(fuzzing)]
568 return LineColumn { line: 0, column: 0 };
569
570 #[cfg(not(fuzzing))]
571 SOURCE_MAP.with(|sm| {
572 let sm = sm.borrow();
573 let fi = sm.fileinfo(*self);
574 fi.offset_line_column(self.hi as usize)
575 })
576 }
577
578 #[cfg(span_locations)]
579 pub(crate) fn file(&self) -> String {
580 #[cfg(fuzzing)]
581 return "<unspecified>".to_owned();
582
583 #[cfg(not(fuzzing))]
584 SOURCE_MAP.with(|sm| {
585 let sm = sm.borrow();
586 sm.filepath(*self)
587 })
588 }
589
590 #[cfg(span_locations)]
591 pub(crate) fn local_file(&self) -> Option<PathBuf> {
592 None
593 }
594
595 #[cfg(not(span_locations))]
596 pub(crate) fn join(&self, _other: Span) -> Option<Span> {
597 Some(Span {})
598 }
599
600 #[cfg(span_locations)]
601 pub(crate) fn join(&self, other: Span) -> Option<Span> {
602 #[cfg(fuzzing)]
603 return {
604 let _ = other;
605 None
606 };
607
608 #[cfg(not(fuzzing))]
609 SOURCE_MAP.with(|sm| {
610 let sm = sm.borrow();
611 // If `other` is not within the same FileInfo as us, return None.
612 if !sm.fileinfo(*self).span_within(other) {
613 return None;
614 }
615 Some(Span {
616 lo: cmp::min(self.lo, other.lo),
617 hi: cmp::max(self.hi, other.hi),
618 })
619 })
620 }
621
622 #[cfg(not(span_locations))]
623 pub(crate) fn source_text(&self) -> Option<String> {
624 None
625 }
626
627 #[cfg(span_locations)]
628 pub(crate) fn source_text(&self) -> Option<String> {
629 #[cfg(fuzzing)]
630 return None;
631
632 #[cfg(not(fuzzing))]
633 {
634 if self.is_call_site() {
635 None
636 } else {
637 Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self)))
638 }
639 }
640 }
641
642 #[cfg(not(span_locations))]
643 pub(crate) fn first_byte(self) -> Self {
644 self
645 }
646
647 #[cfg(span_locations)]
648 pub(crate) fn first_byte(self) -> Self {
649 Span {
650 lo: self.lo,
651 hi: cmp::min(self.lo.saturating_add(1), self.hi),
652 }
653 }
654
655 #[cfg(not(span_locations))]
656 pub(crate) fn last_byte(self) -> Self {
657 self
658 }
659
660 #[cfg(span_locations)]
661 pub(crate) fn last_byte(self) -> Self {
662 Span {
663 lo: cmp::max(self.hi.saturating_sub(1), self.lo),
664 hi: self.hi,
665 }
666 }
667
668 #[cfg(span_locations)]
669 fn is_call_site(&self) -> bool {
670 self.lo == 0 && self.hi == 0
671 }
672}
673
674impl Debug for Span {
675 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
676 #[cfg(span_locations)]
677 return write!(f, "bytes({}..{})", self.lo, self.hi);
678
679 #[cfg(not(span_locations))]
680 write!(f, "Span")
681 }
682}
683
684pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
685 #[cfg(span_locations)]
686 {
687 if span.is_call_site() {
688 return;
689 }
690 }
691
692 if cfg!(span_locations) {
693 debug.field("span", &span);
694 }
695}
696
697#[derive(Clone)]
698pub(crate) struct Group {
699 delimiter: Delimiter,
700 stream: TokenStream,
701 span: Span,
702}
703
704impl Group {
705 pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
706 Group {
707 delimiter,
708 stream,
709 span: Span::call_site(),
710 }
711 }
712
713 pub(crate) fn delimiter(&self) -> Delimiter {
714 self.delimiter
715 }
716
717 pub(crate) fn stream(&self) -> TokenStream {
718 self.stream.clone()
719 }
720
721 pub(crate) fn span(&self) -> Span {
722 self.span
723 }
724
725 pub(crate) fn span_open(&self) -> Span {
726 self.span.first_byte()
727 }
728
729 pub(crate) fn span_close(&self) -> Span {
730 self.span.last_byte()
731 }
732
733 pub(crate) fn set_span(&mut self, span: Span) {
734 self.span = span;
735 }
736}
737
738impl Display for Group {
739 // We attempt to match libproc_macro's formatting.
740 // Empty parens: ()
741 // Nonempty parens: (...)
742 // Empty brackets: []
743 // Nonempty brackets: [...]
744 // Empty braces: { }
745 // Nonempty braces: { ... }
746 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
747 let (open, close) = match self.delimiter {
748 Delimiter::Parenthesis => ("(", ")"),
749 Delimiter::Brace => ("{ ", "}"),
750 Delimiter::Bracket => ("[", "]"),
751 Delimiter::None => ("", ""),
752 };
753
754 f.write_str(open)?;
755 Display::fmt(&self.stream, f)?;
756 if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() {
757 f.write_str(" ")?;
758 }
759 f.write_str(close)?;
760
761 Ok(())
762 }
763}
764
765impl Debug for Group {
766 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
767 let mut debug = fmt.debug_struct("Group");
768 debug.field("delimiter", &self.delimiter);
769 debug.field("stream", &self.stream);
770 debug_span_field_if_nontrivial(&mut debug, self.span);
771 debug.finish()
772 }
773}
774
775#[derive(Clone)]
776pub(crate) struct Ident {
777 sym: Box<str>,
778 span: Span,
779 raw: bool,
780}
781
782impl Ident {
783 #[track_caller]
784 pub(crate) fn new_checked(string: &str, span: Span) -> Self {
785 validate_ident(string);
786 Ident::new_unchecked(string, span)
787 }
788
789 pub(crate) fn new_unchecked(string: &str, span: Span) -> Self {
790 Ident {
791 sym: Box::from(string),
792 span,
793 raw: false,
794 }
795 }
796
797 #[track_caller]
798 pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self {
799 validate_ident_raw(string);
800 Ident::new_raw_unchecked(string, span)
801 }
802
803 pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self {
804 Ident {
805 sym: Box::from(string),
806 span,
807 raw: true,
808 }
809 }
810
811 pub(crate) fn span(&self) -> Span {
812 self.span
813 }
814
815 pub(crate) fn set_span(&mut self, span: Span) {
816 self.span = span;
817 }
818}
819
820pub(crate) fn is_ident_start(c: char) -> bool {
821 c == '_' || c.is_ascii_alphabetic()
822}
823
824pub(crate) fn is_ident_continue(c: char) -> bool {
825 c == '_' || c.is_ascii_alphanumeric()
826}
827
828#[track_caller]
829fn validate_ident(string: &str) {
830 if string.is_empty() {
831 panic!("Ident is not allowed to be empty; use Option<Ident>");
832 }
833
834 if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') {
835 panic!("Ident cannot be a number; use Literal instead");
836 }
837
838 fn ident_ok(string: &str) -> bool {
839 let mut chars = string.chars();
840 let first = chars.next().unwrap();
841 if !is_ident_start(first) {
842 return false;
843 }
844 for ch in chars {
845 if !is_ident_continue(ch) {
846 return false;
847 }
848 }
849 true
850 }
851
852 if !ident_ok(string) {
853 panic!("{:?} is not a valid Ident", string);
854 }
855}
856
857#[track_caller]
858fn validate_ident_raw(string: &str) {
859 validate_ident(string);
860
861 match string {
862 "_" | "super" | "self" | "Self" | "crate" => {
863 panic!("`r#{}` cannot be a raw identifier", string);
864 }
865 _ => {}
866 }
867}
868
869impl PartialEq for Ident {
870 fn eq(&self, other: &Ident) -> bool {
871 self.sym == other.sym && self.raw == other.raw
872 }
873}
874
875impl<T> PartialEq<T> for Ident
876where
877 T: ?Sized + AsRef<str>,
878{
879 fn eq(&self, other: &T) -> bool {
880 let other = other.as_ref();
881 if self.raw {
882 other.starts_with("r#") && *self.sym == other[2..]
883 } else {
884 *self.sym == *other
885 }
886 }
887}
888
889impl Display for Ident {
890 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
891 if self.raw {
892 f.write_str("r#")?;
893 }
894 Display::fmt(&self.sym, f)
895 }
896}
897
898#[allow(clippy::missing_fields_in_debug)]
899impl Debug for Ident {
900 // Ident(proc_macro), Ident(r#union)
901 #[cfg(not(span_locations))]
902 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
903 let mut debug = f.debug_tuple("Ident");
904 debug.field(&format_args!("{}", self));
905 debug.finish()
906 }
907
908 // Ident {
909 // sym: proc_macro,
910 // span: bytes(128..138)
911 // }
912 #[cfg(span_locations)]
913 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
914 let mut debug = f.debug_struct("Ident");
915 debug.field("sym", &format_args!("{}", self));
916 debug_span_field_if_nontrivial(&mut debug, self.span);
917 debug.finish()
918 }
919}
920
921#[derive(Clone)]
922pub(crate) struct Literal {
923 pub(crate) repr: String,
924 span: Span,
925}
926
927macro_rules! suffixed_numbers {
928 ($($name:ident => $kind:ident,)*) => ($(
929 pub(crate) fn $name(n: $kind) -> Literal {
930 Literal::_new(format!(concat!("{}", stringify!($kind)), n))
931 }
932 )*)
933}
934
935macro_rules! unsuffixed_numbers {
936 ($($name:ident => $kind:ident,)*) => ($(
937 pub(crate) fn $name(n: $kind) -> Literal {
938 Literal::_new(n.to_string())
939 }
940 )*)
941}
942
943impl Literal {
944 pub(crate) fn _new(repr: String) -> Self {
945 Literal {
946 repr,
947 span: Span::call_site(),
948 }
949 }
950
951 pub(crate) fn from_str_checked(repr: &str) -> Result<Self, LexError> {
952 let mut cursor = get_cursor(repr);
953 #[cfg(span_locations)]
954 let lo = cursor.off;
955
956 let negative = cursor.starts_with_char('-');
957 if negative {
958 cursor = cursor.advance(1);
959 if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) {
960 return Err(LexError::call_site());
961 }
962 }
963
964 if let Ok((rest, mut literal)) = parse::literal(cursor) {
965 if rest.is_empty() {
966 if negative {
967 literal.repr.insert(0, '-');
968 }
969 literal.span = Span {
970 #[cfg(span_locations)]
971 lo,
972 #[cfg(span_locations)]
973 hi: rest.off,
974 };
975 return Ok(literal);
976 }
977 }
978 Err(LexError::call_site())
979 }
980
981 pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self {
982 Literal::_new(repr.to_owned())
983 }
984
985 suffixed_numbers! {
986 u8_suffixed => u8,
987 u16_suffixed => u16,
988 u32_suffixed => u32,
989 u64_suffixed => u64,
990 u128_suffixed => u128,
991 usize_suffixed => usize,
992 i8_suffixed => i8,
993 i16_suffixed => i16,
994 i32_suffixed => i32,
995 i64_suffixed => i64,
996 i128_suffixed => i128,
997 isize_suffixed => isize,
998
999 f32_suffixed => f32,
1000 f64_suffixed => f64,
1001 }
1002
1003 unsuffixed_numbers! {
1004 u8_unsuffixed => u8,
1005 u16_unsuffixed => u16,
1006 u32_unsuffixed => u32,
1007 u64_unsuffixed => u64,
1008 u128_unsuffixed => u128,
1009 usize_unsuffixed => usize,
1010 i8_unsuffixed => i8,
1011 i16_unsuffixed => i16,
1012 i32_unsuffixed => i32,
1013 i64_unsuffixed => i64,
1014 i128_unsuffixed => i128,
1015 isize_unsuffixed => isize,
1016 }
1017
1018 pub(crate) fn f32_unsuffixed(f: f32) -> Literal {
1019 let mut s = f.to_string();
1020 if !s.contains('.') {
1021 s.push_str(".0");
1022 }
1023 Literal::_new(s)
1024 }
1025
1026 pub(crate) fn f64_unsuffixed(f: f64) -> Literal {
1027 let mut s = f.to_string();
1028 if !s.contains('.') {
1029 s.push_str(".0");
1030 }
1031 Literal::_new(s)
1032 }
1033
1034 pub(crate) fn string(string: &str) -> Literal {
1035 let mut repr = String::with_capacity(string.len() + 2);
1036 repr.push('"');
1037 escape_utf8(string, &mut repr);
1038 repr.push('"');
1039 Literal::_new(repr)
1040 }
1041
1042 pub(crate) fn character(ch: char) -> Literal {
1043 let mut repr = String::new();
1044 repr.push('\'');
1045 if ch == '"' {
1046 // escape_debug turns this into '\"' which is unnecessary.
1047 repr.push(ch);
1048 } else {
1049 repr.extend(ch.escape_debug());
1050 }
1051 repr.push('\'');
1052 Literal::_new(repr)
1053 }
1054
1055 pub(crate) fn byte_character(byte: u8) -> Literal {
1056 let mut repr = "b'".to_string();
1057 #[allow(clippy::match_overlapping_arm)]
1058 match byte {
1059 b'\0' => repr.push_str(r"\0"),
1060 b'\t' => repr.push_str(r"\t"),
1061 b'\n' => repr.push_str(r"\n"),
1062 b'\r' => repr.push_str(r"\r"),
1063 b'\'' => repr.push_str(r"\'"),
1064 b'\\' => repr.push_str(r"\\"),
1065 b'\x20'..=b'\x7E' => repr.push(byte as char),
1066 _ => {
1067 let _ = write!(repr, r"\x{:02X}", byte);
1068 }
1069 }
1070 repr.push('\'');
1071 Literal::_new(repr)
1072 }
1073
1074 pub(crate) fn byte_string(bytes: &[u8]) -> Literal {
1075 let mut repr = "b\"".to_string();
1076 let mut bytes = bytes.iter();
1077 while let Some(&b) = bytes.next() {
1078 #[allow(clippy::match_overlapping_arm)]
1079 match b {
1080 b'\0' => repr.push_str(match bytes.as_slice().first() {
1081 // circumvent clippy::octal_escapes lint
1082 Some(b'0'..=b'7') => r"\x00",
1083 _ => r"\0",
1084 }),
1085 b'\t' => repr.push_str(r"\t"),
1086 b'\n' => repr.push_str(r"\n"),
1087 b'\r' => repr.push_str(r"\r"),
1088 b'"' => repr.push_str("\\\""),
1089 b'\\' => repr.push_str(r"\\"),
1090 b'\x20'..=b'\x7E' => repr.push(b as char),
1091 _ => {
1092 let _ = write!(repr, r"\x{:02X}", b);
1093 }
1094 }
1095 }
1096 repr.push('"');
1097 Literal::_new(repr)
1098 }
1099
1100 pub(crate) fn c_string(string: &CStr) -> Literal {
1101 let mut repr = "c\"".to_string();
1102 let mut bytes = string.to_bytes();
1103 while !bytes.is_empty() {
1104 let (valid, invalid) = match str::from_utf8(bytes) {
1105 Ok(all_valid) => {
1106 bytes = b"";
1107 (all_valid, bytes)
1108 }
1109 Err(utf8_error) => {
1110 let (valid, rest) = bytes.split_at(utf8_error.valid_up_to());
1111 let valid = str::from_utf8(valid).unwrap();
1112 let invalid = utf8_error
1113 .error_len()
1114 .map_or(rest, |error_len| &rest[..error_len]);
1115 bytes = &bytes[valid.len() + invalid.len()..];
1116 (valid, invalid)
1117 }
1118 };
1119 escape_utf8(valid, &mut repr);
1120 for &byte in invalid {
1121 let _ = write!(repr, r"\x{:02X}", byte);
1122 }
1123 }
1124 repr.push('"');
1125 Literal::_new(repr)
1126 }
1127
1128 pub(crate) fn span(&self) -> Span {
1129 self.span
1130 }
1131
1132 pub(crate) fn set_span(&mut self, span: Span) {
1133 self.span = span;
1134 }
1135
1136 pub(crate) fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
1137 #[cfg(not(span_locations))]
1138 {
1139 let _ = range;
1140 None
1141 }
1142
1143 #[cfg(span_locations)]
1144 {
1145 use core::ops::Bound;
1146
1147 let lo = match range.start_bound() {
1148 Bound::Included(start) => {
1149 let start = u32::try_from(*start).ok()?;
1150 self.span.lo.checked_add(start)?
1151 }
1152 Bound::Excluded(start) => {
1153 let start = u32::try_from(*start).ok()?;
1154 self.span.lo.checked_add(start)?.checked_add(1)?
1155 }
1156 Bound::Unbounded => self.span.lo,
1157 };
1158 let hi = match range.end_bound() {
1159 Bound::Included(end) => {
1160 let end = u32::try_from(*end).ok()?;
1161 self.span.lo.checked_add(end)?.checked_add(1)?
1162 }
1163 Bound::Excluded(end) => {
1164 let end = u32::try_from(*end).ok()?;
1165 self.span.lo.checked_add(end)?
1166 }
1167 Bound::Unbounded => self.span.hi,
1168 };
1169 if lo <= hi && hi <= self.span.hi {
1170 Some(Span { lo, hi })
1171 } else {
1172 None
1173 }
1174 }
1175 }
1176}
1177
1178impl Display for Literal {
1179 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1180 Display::fmt(&self.repr, f)
1181 }
1182}
1183
1184impl Debug for Literal {
1185 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1186 let mut debug = fmt.debug_struct("Literal");
1187 debug.field("lit", &format_args!("{}", self.repr));
1188 debug_span_field_if_nontrivial(&mut debug, self.span);
1189 debug.finish()
1190 }
1191}
1192
1193fn escape_utf8(string: &str, repr: &mut String) {
1194 let mut chars = string.chars();
1195 while let Some(ch) = chars.next() {
1196 if ch == '\0' {
1197 repr.push_str(
1198 if chars
1199 .as_str()
1200 .starts_with(|next| '0' <= next && next <= '7')
1201 {
1202 // circumvent clippy::octal_escapes lint
1203 r"\x00"
1204 } else {
1205 r"\0"
1206 },
1207 );
1208 } else if ch == '\'' {
1209 // escape_debug turns this into "\'" which is unnecessary.
1210 repr.push(ch);
1211 } else {
1212 repr.extend(ch.escape_debug());
1213 }
1214 }
1215}
1216
1217#[cfg(feature = "proc-macro")]
1218pub(crate) trait FromStr2: FromStr<Err = proc_macro::LexError> {
1219 #[cfg(wrap_proc_macro)]
1220 fn valid(src: &str) -> bool;
1221
1222 #[cfg(wrap_proc_macro)]
1223 fn from_str_checked(src: &str) -> Result<Self, imp::LexError> {
1224 // Validate using fallback parser, because rustc is incapable of
1225 // returning a recoverable Err for certain invalid token streams, and
1226 // will instead permanently poison the compilation.
1227 if !Self::valid(src) {
1228 return Err(imp::LexError::CompilerPanic);
1229 }
1230
1231 // Catch panic to work around https://github.com/rust-lang/rust/issues/58736.
1232 match panic::catch_unwind(|| Self::from_str(src)) {
1233 Ok(Ok(ok)) => Ok(ok),
1234 Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)),
1235 Err(_panic) => Err(imp::LexError::CompilerPanic),
1236 }
1237 }
1238
1239 fn from_str_unchecked(src: &str) -> Self {
1240 Self::from_str(src).unwrap()
1241 }
1242}
1243
1244#[cfg(feature = "proc-macro")]
1245impl FromStr2 for proc_macro::TokenStream {
1246 #[cfg(wrap_proc_macro)]
1247 fn valid(src: &str) -> bool {
1248 TokenStream::from_str_checked(src).is_ok()
1249 }
1250}
1251
1252#[cfg(feature = "proc-macro")]
1253impl FromStr2 for proc_macro::Literal {
1254 #[cfg(wrap_proc_macro)]
1255 fn valid(src: &str) -> bool {
1256 Literal::from_str_checked(src).is_ok()
1257 }
1258}