Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

a4fdb2a4 · Linus Torvalds <[email protected]> · 2015-09-04 07:18:09 -07:00

Pull arm64 updates from Will Deacon:

 - Support for new architectural features introduced in ARMv8.1:
   * Privileged Access Never (PAN) to catch user pointer dereferences in
     the kernel
   * Large System Extension (LSE) for building scalable atomics and locks
     (depends on locking/arch-atomic from tip, which is included here)
   * Hardware Dirty Bit Management (DBM) for updating clean PTEs
     automatically

 - Move our PSCI implementation out into drivers/firmware/, where it can
   be shared with arch/arm/. RMK has also pulled this component branch
   and has additional patches moving arch/arm/ over. MAINTAINERS is
   updated accordingly.

 - Better BUG implementation based on the BRK instruction for trapping

 - Leaf TLB invalidation for unmapping user pages

 - Support for PROBE_ONLY PCI configurations

 - Various cleanups and non-critical fixes, including:
   * Always flush FP/SIMD state over exec()
   * Restrict memblock additions based on range of linear mapping
   * Ensure *(LIST_POISON) generates a fatal fault
   * Context-tracking syscall return no longer corrupts return value when
     not forced on.
   * Alternatives patching synchronisation/stability improvements
   * Signed sub-word cmpxchg compare fix (tickled by HAVE_CMPXCHG_LOCAL)
   * Force SMP=y
   * Hide direct DCC access from userspace
   * Fix EFI stub memory allocation when DRAM starts at 0x0

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (92 commits)
  arm64: flush FP/SIMD state correctly after execve()
  arm64: makefile: fix perf_callchain.o kconfig dependency
  arm64: set MAX_MEMBLOCK_ADDR according to linear region size
  of/fdt: make memblock maximum physical address arch configurable
  arm64: Fix source code file path in comments
  arm64: entry: always restore x0 from the stack on syscall return
  arm64: mdscr_el1: avoid exposing DCC to userspace
  arm64: kconfig: Move LIST_POISON to a safe value
  arm64: Add __exception_irq_entry definition for function graph
  arm64: mm: ensure patched kernel text is fetched from PoU
  arm64: alternatives: ensure secondary CPUs execute ISB after patching
  arm64: make ll/sc __cmpxchg_case_##name asm consistent
  arm64: dma-mapping: Simplify pgprot handling
  arm64: restore cpu suspend/resume functionality
  ARM64: PCI: do not enable resources on PROBE_ONLY systems
  arm64: cmpxchg: truncate sub-word signed types before comparison
  arm64: alternative: put secondary CPUs into polling loop during patch
  arm64/Documentation: clarify wording regarding memory below the Image
  arm64: lse: fix lse cmpxchg code indentation
  arm64: remove redundant object file list
  ...

Commit: a4fdb2a46f617b8b2cd47acec026ec16532edbc6
Parents: 807249d3 · 674c242c

Merge commit. Changes below are compared with the first parent: 807249d3.

Changes

86 files changed · +2321 −1145

File	Status	Add	Del	Content
Documentation/arm64/booting.txt	modified	+7	−4	View file
arch/arm64/Kconfig	modified	+64	−22	View file
arch/arm64/Makefile	modified	+16	−2	View file
arch/arm64/boot/Makefile	modified	+12	−0	View file
arch/arm64/include/asm/alternative.h	modified	+74	−4	View file
arch/arm64/include/asm/assembler.h	modified	+0	−14	View file
arch/arm64/include/asm/atomic.h	modified	+52	−227	View file
arch/arm64/include/asm/atomic_ll_sc.h	added	+247	−0	View file
arch/arm64/include/asm/atomic_lse.h	added	+391	−0	View file
arch/arm64/include/asm/barrier.h	modified	+0	−24	View file
arch/arm64/include/asm/bug.h	added	+64	−0	View file
arch/arm64/include/asm/cmpxchg.h	modified	+74	−118	View file
arch/arm64/include/asm/cpufeature.h	modified	+15	−3	View file
arch/arm64/include/asm/cputype.h	modified	+0	−3	View file
arch/arm64/include/asm/debug-monitors.h	modified	+16	−28	View file
arch/arm64/include/asm/esr.h	modified	+9	−0	View file
arch/arm64/include/asm/exception.h	modified	+6	−0	View file
arch/arm64/include/asm/fixmap.h	modified	+1	−1	View file
arch/arm64/include/asm/futex.h	modified	+10	−0	View file
arch/arm64/include/asm/hardirq.h	modified	+0	−4	View file
arch/arm64/include/asm/hugetlb.h	modified	+0	−4	View file
arch/arm64/include/asm/irq_work.h	modified	+0	−11	View file
arch/arm64/include/asm/lse.h	added	+53	−0	View file
arch/arm64/include/asm/memory.h	modified	+8	−0	View file
arch/arm64/include/asm/mmu.h	modified	+0	−1	View file
arch/arm64/include/asm/percpu.h	modified	+0	−8	View file
arch/arm64/include/asm/perf_event.h	modified	+1	−1	View file
arch/arm64/include/asm/pgtable-hwdef.h	modified	+3	−0	View file
arch/arm64/include/asm/pgtable.h	modified	+160	−7	View file
arch/arm64/include/asm/processor.h	modified	+2	−0	View file
arch/arm64/include/asm/ptrace.h	modified	+0	−4	View file
arch/arm64/include/asm/smp.h	modified	+0	−4	View file
arch/arm64/include/asm/smp_plat.h	modified	+0	−2	View file
arch/arm64/include/asm/spinlock.h	modified	+114	−33	View file
arch/arm64/include/asm/spinlock_types.h	modified	+2	−0	View file
arch/arm64/include/asm/sysreg.h	modified	+35	−5	View file
arch/arm64/include/asm/tlb.h	modified	+6	−1	View file
arch/arm64/include/asm/tlbflush.h	modified	+34	−42	View file
arch/arm64/include/asm/topology.h	modified	+0	−9	View file
arch/arm64/include/asm/traps.h	modified	+21	−2	View file
arch/arm64/include/asm/uaccess.h	modified	+11	−0	View file
arch/arm64/include/uapi/asm/hwcap.h	modified	+1	−0	View file
arch/arm64/include/uapi/asm/ptrace.h	modified	+1	−0	View file
arch/arm64/kernel/Makefile	modified	+3	−3	View file
arch/arm64/kernel/alternative.c	modified	+25	−5	View file
arch/arm64/kernel/armv8_deprecated.c	modified	+8	−11	View file
arch/arm64/kernel/cpu_ops.c	modified	+0	−2	View file
arch/arm64/kernel/cpufeature.c	modified	+46	−7	View file
arch/arm64/kernel/debug-monitors.c	modified	+2	−2	View file
arch/arm64/kernel/efi-stub.c	modified	+32	−9	View file
arch/arm64/kernel/entry.S	modified	+19	−27	View file
arch/arm64/kernel/fpsimd.c	modified	+1	−0	View file
arch/arm64/kernel/head.S	modified	+8	−7	View file
arch/arm64/kernel/hw_breakpoint.c	modified	+3	−3	View file
arch/arm64/kernel/insn.c	modified	+2	−3	View file
arch/arm64/kernel/irq.c	modified	+0	−2	View file
arch/arm64/kernel/kgdb.c	modified	+6	−6	View file
arch/arm64/kernel/pci.c	modified	+13	−0	View file
arch/arm64/kernel/perf_callchain.c	added	+196	−0	View file
arch/arm64/kernel/perf_event.c	modified	+25	−285	View file
arch/arm64/kernel/psci.c	modified	+0	−5	View file
arch/arm64/kernel/ptrace.c	modified	+91	−1	View file
arch/arm64/kernel/setup.c	modified	+34	−51	View file
arch/arm64/kernel/sleep.S	modified	+8	−6	View file
arch/arm64/kernel/smp.c	modified	+14	−1	View file
arch/arm64/kernel/time.c	modified	+0	−2	View file
arch/arm64/kernel/topology.c	modified	+1	−1	View file
arch/arm64/kernel/traps.c	modified	+80	−14	View file
arch/arm64/kvm/hyp.S	modified	+10	−2	View file
arch/arm64/lib/Makefile	modified	+13	−0	View file
arch/arm64/lib/atomic_ll_sc.c	added	+3	−0	View file
arch/arm64/lib/bitops.S	modified	+26	−19	View file
arch/arm64/lib/clear_user.S	modified	+8	−0	View file
arch/arm64/lib/copy_from_user.S	modified	+19	−6	View file
arch/arm64/lib/copy_in_user.S	modified	+19	−6	View file
arch/arm64/lib/copy_to_user.S	modified	+19	−6	View file
arch/arm64/mm/cache.S	modified	+6	−1	View file
arch/arm64/mm/context.c	modified	+0	−16	View file
arch/arm64/mm/dma-mapping.c	modified	+20	−13	View file
arch/arm64/mm/fault.c	modified	+26	−2	View file
arch/arm64/mm/flush.c	modified	+0	−4	View file
arch/arm64/mm/hugetlbpage.c	modified	+0	−4	View file
arch/arm64/mm/init.c	modified	+2	−2	View file
arch/arm64/mm/mmu.c	modified	+1	−12	View file
arch/arm64/mm/proc.S	modified	+15	−6	View file
drivers/of/fdt.c	modified	+7	−5	View file

Only the first 600 diff lines are shown. Download the complete patch.

Documentation/arm64/booting.txt

View file

diff --git a/Documentation/arm64/booting.txt b/Documentation/arm64/booting.txt
index 1690350..7d9d3c2 100644
--- a/Documentation/arm64/booting.txt
+++ b/Documentation/arm64/booting.txt
@@ -81,7 +81,7 @@ The decompressed kernel image contains a 64-byte header as follows:
   u64 res3	= 0;		/* reserved */
   u64 res4	= 0;		/* reserved */
   u32 magic	= 0x644d5241;	/* Magic number, little endian, "ARM\x64" */
-  u32 res5;      		/* reserved (used for PE COFF offset) */
+  u32 res5;			/* reserved (used for PE COFF offset) */
 
 
 Header notes:
@@ -103,7 +103,7 @@ Header notes:
 
 - The flags field (introduced in v3.17) is a little-endian 64-bit field
   composed as follows:
-  Bit 0: 	Kernel endianness.  1 if BE, 0 if LE.
+  Bit 0:	Kernel endianness.  1 if BE, 0 if LE.
   Bits 1-63:	Reserved.
 
 - When image_size is zero, a bootloader should attempt to keep as much
@@ -115,11 +115,14 @@ The Image must be placed text_offset bytes from a 2MB aligned base
 address near the start of usable system RAM and called there. Memory
 below that base address is currently unusable by Linux, and therefore it
 is strongly recommended that this location is the start of system RAM.
+The region between the 2 MB aligned base address and the start of the
+image has no special significance to the kernel, and may be used for
+other purposes.
 At least image_size bytes from the start of the image must be free for
 use by the kernel.
 
-Any memory described to the kernel (even that below the 2MB aligned base
-address) which is not marked as reserved from the kernel e.g. with a
+Any memory described to the kernel (even that below the start of the
+image) which is not marked as reserved from the kernel (e.g., with a
 memreserve region in the device tree) will be considered as available to
 the kernel.
 

arch/arm64/Kconfig

View file

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 51832ad..7d95663 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -29,7 +29,7 @@ config ARM64
 	select EDAC_SUPPORT
 	select GENERIC_ALLOCATOR
 	select GENERIC_CLOCKEVENTS
-	select GENERIC_CLOCKEVENTS_BROADCAST if SMP
+	select GENERIC_CLOCKEVENTS_BROADCAST
 	select GENERIC_CPU_AUTOPROBE
 	select GENERIC_EARLY_IOREMAP
 	select GENERIC_IRQ_PROBE
@@ -54,6 +54,7 @@ config ARM64
 	select HAVE_C_RECORDMCOUNT
 	select HAVE_CC_STACKPROTECTOR
 	select HAVE_CMPXCHG_DOUBLE
+	select HAVE_CMPXCHG_LOCAL
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_API_DEBUG
@@ -105,6 +106,10 @@ config NO_IOPORT_MAP
 config STACKTRACE_SUPPORT
 	def_bool y
 
+config ILLEGAL_POINTER_VALUE
+	hex
+	default 0xdead000000000000
+
 config LOCKDEP_SUPPORT
 	def_bool y
 
@@ -114,6 +119,14 @@ config TRACE_IRQFLAGS_SUPPORT
 config RWSEM_XCHGADD_ALGORITHM
 	def_bool y
 
+config GENERIC_BUG
+	def_bool y
+	depends on BUG
+
+config GENERIC_BUG_RELATIVE_POINTERS
+	def_bool y
+	depends on GENERIC_BUG
+
 config GENERIC_HWEIGHT
 	def_bool y
 
@@ -138,6 +151,9 @@ config NEED_DMA_MAP_STATE
 config NEED_SG_DMA_LENGTH
 	def_bool y
 
+config SMP
+	def_bool y
+
 config SWIOTLB
 	def_bool y
 
@@ -372,22 +388,8 @@ config CPU_BIG_ENDIAN
        help
          Say Y if you plan on running a kernel in big-endian mode.
 
-config SMP
-	bool "Symmetric Multi-Processing"
-	help
-	  This enables support for systems with more than one CPU.  If
-	  you say N here, the kernel will run on single and
-	  multiprocessor machines, but will use only one CPU of a
-	  multiprocessor machine. If you say Y here, the kernel will run
-	  on many, but not all, single processor machines. On a single
-	  processor machine, the kernel will run faster if you say N
-	  here.
-
-	  If you don't know what to do here, say N.
-
 config SCHED_MC
 	bool "Multi-core scheduler support"
-	depends on SMP
 	help
 	  Multi-core scheduler support improves the CPU scheduler's decision
 	  making when dealing with multi-core CPU chips at a cost of slightly
@@ -395,7 +397,6 @@ config SCHED_MC
 
 config SCHED_SMT
 	bool "SMT scheduler support"
-	depends on SMP
 	help
 	  Improves the CPU scheduler's decision making when dealing with
 	  MultiThreading at a cost of slightly increased overhead in some
@@ -404,23 +405,17 @@ config SCHED_SMT
 config NR_CPUS
 	int "Maximum number of CPUs (2-4096)"
 	range 2 4096
-	depends on SMP
 	# These have to remain sorted largest to smallest
 	default "64"
 
 config HOTPLUG_CPU
 	bool "Support for hot-pluggable CPUs"
-	depends on SMP
 	help
 	  Say Y here to experiment with turning CPUs off and on.  CPUs
 	  can be controlled through /sys/devices/system/cpu.
 
 source kernel/Kconfig.preempt
 
-config UP_LATE_INIT
-       def_bool y
-       depends on !SMP
-
 config HZ
 	int
 	default 100
@@ -562,6 +557,53 @@ config SETEND_EMULATION
 	  If unsure, say Y
 endif
 
+menu "ARMv8.1 architectural features"
+
+config ARM64_HW_AFDBM
+	bool "Support for hardware updates of the Access and Dirty page flags"
+	default y
+	help
+	  The ARMv8.1 architecture extensions introduce support for
+	  hardware updates of the access and dirty information in page
+	  table entries. When enabled in TCR_EL1 (HA and HD bits) on
+	  capable processors, accesses to pages with PTE_AF cleared will
+	  set this bit instead of raising an access flag fault.
+	  Similarly, writes to read-only pages with the DBM bit set will
+	  clear the read-only bit (AP[2]) instead of raising a
+	  permission fault.
+
+	  Kernels built with this configuration option enabled continue
+	  to work on pre-ARMv8.1 hardware and the performance impact is
+	  minimal. If unsure, say Y.
+
+config ARM64_PAN
+	bool "Enable support for Privileged Access Never (PAN)"
+	default y
+	help
+	 Privileged Access Never (PAN; part of the ARMv8.1 Extensions)
+	 prevents the kernel or hypervisor from accessing user-space (EL0)
+	 memory directly.
+
+	 Choosing this option will cause any unprotected (not using
+	 copy_to_user et al) memory access to fail with a permission fault.
+
+	 The feature is detected at runtime, and will remain as a 'nop'
+	 instruction if the cpu does not implement the feature.
+
+config ARM64_LSE_ATOMICS
+	bool "Atomic instructions"
+	help
+	  As part of the Large System Extensions, ARMv8.1 introduces new
+	  atomic instructions that are designed specifically to scale in
+	  very large systems.
+
+	  Say Y here to make use of these instructions for the in-kernel
+	  atomic routines. This incurs a small overhead on CPUs that do
+	  not support these instructions and requires the kernel to be
+	  built with binutils >= 2.25.
+
+endmenu
+
 endmenu
 
 menu "Boot options"

arch/arm64/Makefile

View file

diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index 4d2a925..15ff5b4 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -17,7 +17,18 @@ GZFLAGS		:=-9
 
 KBUILD_DEFCONFIG := defconfig
 
-KBUILD_CFLAGS	+= -mgeneral-regs-only
+# Check for binutils support for specific extensions
+lseinstr := $(call as-instr,.arch_extension lse,-DCONFIG_AS_LSE=1)
+
+ifeq ($(CONFIG_ARM64_LSE_ATOMICS), y)
+  ifeq ($(lseinstr),)
+$(warning LSE atomics not supported by binutils)
+  endif
+endif
+
+KBUILD_CFLAGS	+= -mgeneral-regs-only $(lseinstr)
+KBUILD_AFLAGS	+= $(lseinstr)
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 AS		+= -EB
@@ -58,7 +69,10 @@ all:	$(KBUILD_IMAGE) $(KBUILD_DTBS)
 
 boot := arch/arm64/boot
 
-Image Image.gz: vmlinux
+Image: vmlinux
+	$(Q)$(MAKE) $(build)=$(boot) $(boot)/$@
+
+Image.%: vmlinux
 	$(Q)$(MAKE) $(build)=$(boot) $(boot)/$@
 
 zinstall install: vmlinux

arch/arm64/boot/Makefile

View file

diff --git a/arch/arm64/boot/Makefile b/arch/arm64/boot/Makefile
index 5a0e3ab..abcbba2 100644
--- a/arch/arm64/boot/Makefile
+++ b/arch/arm64/boot/Makefile
@@ -19,9 +19,21 @@ targets := Image Image.gz
 $(obj)/Image: vmlinux FORCE
 	$(call if_changed,objcopy)
 
+$(obj)/Image.bz2: $(obj)/Image FORCE
+	$(call if_changed,bzip2)
+
 $(obj)/Image.gz: $(obj)/Image FORCE
 	$(call if_changed,gzip)
 
+$(obj)/Image.lz4: $(obj)/Image FORCE
+	$(call if_changed,lz4)
+
+$(obj)/Image.lzma: $(obj)/Image FORCE
+	$(call if_changed,lzma)
+
+$(obj)/Image.lzo: $(obj)/Image FORCE
+	$(call if_changed,lzo)
+
 install: $(obj)/Image
 	$(CONFIG_SHELL) $(srctree)/$(src)/install.sh $(KERNELRELEASE) \
 	$(obj)/Image System.map "$(INSTALL_PATH)"

arch/arm64/include/asm/alternative.h

View file

diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h
index c385a0c..d56ec07 100644
--- a/arch/arm64/include/asm/alternative.h
+++ b/arch/arm64/include/asm/alternative.h
@@ -3,6 +3,8 @@
 
 #ifndef __ASSEMBLY__
 
+#include <linux/init.h>
+#include <linux/kconfig.h>
 #include <linux/types.h>
 #include <linux/stddef.h>
 #include <linux/stringify.h>
@@ -15,7 +17,7 @@ struct alt_instr {
 	u8  alt_len;		/* size of new instruction(s), <= orig_len */
 };
 
-void apply_alternatives_all(void);
+void __init apply_alternatives_all(void);
 void apply_alternatives(void *start, size_t length);
 void free_alternatives_memory(void);
 
@@ -40,7 +42,8 @@ void free_alternatives_memory(void);
  * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
  * containing commit 4e4d08cf7399b606 or c1baaddf8861).
  */
-#define ALTERNATIVE(oldinstr, newinstr, feature)			\
+#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled)	\
+	".if "__stringify(cfg_enabled)" == 1\n"				\
 	"661:\n\t"							\
 	oldinstr "\n"							\
 	"662:\n"							\
@@ -53,7 +56,11 @@ void free_alternatives_memory(void);
 	"664:\n\t"							\
 	".popsection\n\t"						\
 	".org	. - (664b-663b) + (662b-661b)\n\t"			\
-	".org	. - (662b-661b) + (664b-663b)\n"
+	".org	. - (662b-661b) + (664b-663b)\n"			\
+	".endif\n"
+
+#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...)	\
+	__ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
 
 #else
 
@@ -65,7 +72,8 @@ void free_alternatives_memory(void);
 	.byte \alt_len
 .endm
 
-.macro alternative_insn insn1 insn2 cap
+.macro alternative_insn insn1, insn2, cap, enable = 1
+	.if \enable
 661:	\insn1
 662:	.pushsection .altinstructions, "a"
 	altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
@@ -75,8 +83,70 @@ void free_alternatives_memory(void);
 664:	.popsection
 	.org	. - (664b-663b) + (662b-661b)
 	.org	. - (662b-661b) + (664b-663b)
+	.endif
+.endm
+
+/*
+ * Begin an alternative code sequence.
+ *
+ * The code that follows this macro will be assembled and linked as
+ * normal. There are no restrictions on this code.
+ */
+.macro alternative_if_not cap, enable = 1
+	.if \enable
+	.pushsection .altinstructions, "a"
+	altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
+	.popsection
+661:
+	.endif
+.endm
+
+/*
+ * Provide the alternative code sequence.
+ *
+ * The code that follows this macro is assembled into a special
+ * section to be used for dynamic patching. Code that follows this
+ * macro must:
+ *
+ * 1. Be exactly the same length (in bytes) as the default code
+ *    sequence.
+ *
+ * 2. Not contain a branch target that is used outside of the
+ *    alternative sequence it is defined in (branches into an
+ *    alternative sequence are not fixed up).
+ */
+.macro alternative_else, enable = 1
+	.if \enable
+662:	.pushsection .altinstr_replacement, "ax"
+663:
+	.endif
 .endm
 
+/*
+ * Complete an alternative code sequence.
+ */
+.macro alternative_endif, enable = 1
+	.if \enable
+664:	.popsection
+	.org	. - (664b-663b) + (662b-661b)
+	.org	. - (662b-661b) + (664b-663b)
+	.endif
+.endm
+
+#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)	\
+	alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
+
+
 #endif  /*  __ASSEMBLY__  */
 
+/*
+ * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
+ *
+ * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
+ * N.B. If CONFIG_FOO is specified, but not selected, the whole block
+ *      will be omitted, including oldinstr.
+ */
+#define ALTERNATIVE(oldinstr, newinstr, ...)   \
+	_ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
+
 #endif /* __ASM_ALTERNATIVE_H */

arch/arm64/include/asm/assembler.h

View file

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 144b64a..b51f2cc 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -50,18 +50,6 @@
 	.endm
 
 /*
- * Save/disable and restore interrupts.
- */
-	.macro	save_and_disable_irqs, olddaif
-	mrs	\olddaif, daif
-	disable_irq
-	.endm
-
-	.macro	restore_irqs, olddaif
-	msr	daif, \olddaif
-	.endm
-
-/*
  * Enable and disable debug exceptions.
  */
 	.macro	disable_dbg
@@ -103,9 +91,7 @@
  * SMP data memory barrier
  */
 	.macro	smp_dmb, opt
-#ifdef CONFIG_SMP
 	dmb	\opt
-#endif
 	.endm
 
 #define USER(l, x...)				\

arch/arm64/include/asm/atomic.h

View file

diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h
index 866a71f..35a6778 100644
--- a/arch/arm64/include/asm/atomic.h
+++ b/arch/arm64/include/asm/atomic.h
@@ -24,247 +24,72 @@
 #include <linux/types.h>
 
 #include <asm/barrier.h>
-#include <asm/cmpxchg.h>
-
-#define ATOMIC_INIT(i)	{ (i) }
+#include <asm/lse.h>
 
 #ifdef __KERNEL__
 
-/*
- * On ARM, ordinary assignment (str instruction) doesn't clear the local
- * strex/ldrex monitor on some implementations. The reason we can use it for
- * atomic_set() is the clrex or dummy strex done on every exception return.
- */
-#define atomic_read(v)	ACCESS_ONCE((v)->counter)
-#define atomic_set(v,i)	(((v)->counter) = (i))
-
-/*
- * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
- * store exclusive to ensure that these are atomic.  We may loop
- * to ensure that the update happens.
- */
-
-#define ATOMIC_OP(op, asm_op)						\
-static inline void atomic_##op(int i, atomic_t *v)			\
-{									\
-	unsigned long tmp;						\
-	int result;							\
-									\
-	asm volatile("// atomic_" #op "\n"				\
-"1:	ldxr	%w0, %2\n"						\
-"	" #asm_op "	%w0, %w0, %w3\n"				\
-"	stxr	%w1, %w0, %2\n"						\
-"	cbnz	%w1, 1b"						\
-	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
-	: "Ir" (i));							\
-}									\
-
-#define ATOMIC_OP_RETURN(op, asm_op)					\
-static inline int atomic_##op##_return(int i, atomic_t *v)		\
-{									\
-	unsigned long tmp;						\
-	int result;							\
-									\
-	asm volatile("// atomic_" #op "_return\n"			\
-"1:	ldxr	%w0, %2\n"						\
-"	" #asm_op "	%w0, %w0, %w3\n"				\
-"	stlxr	%w1, %w0, %2\n"						\
-"	cbnz	%w1, 1b"						\
-	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
-	: "Ir" (i)							\
-	: "memory");							\
-									\
-	smp_mb();							\
-	return result;							\
-}
+#define __ARM64_IN_ATOMIC_IMPL
 
-#define ATOMIC_OPS(op, asm_op)						\
-	ATOMIC_OP(op, asm_op)						\
-	ATOMIC_OP_RETURN(op, asm_op)
-
-ATOMIC_OPS(add, add)
-ATOMIC_OPS(sub, sub)
-
-#define atomic_andnot atomic_andnot
-
-ATOMIC_OP(and, and)
-ATOMIC_OP(andnot, bic)
-ATOMIC_OP(or, orr)
-ATOMIC_OP(xor, eor)
-
-#undef ATOMIC_OPS
-#undef ATOMIC_OP_RETURN
-#undef ATOMIC_OP
-
-static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
-{
-	unsigned long tmp;
-	int oldval;
-
-	smp_mb();
-
-	asm volatile("// atomic_cmpxchg\n"
-"1:	ldxr	%w1, %2\n"
-"	cmp	%w1, %w3\n"
-"	b.ne	2f\n"
-"	stxr	%w0, %w4, %2\n"
-"	cbnz	%w0, 1b\n"
-"2:"
-	: "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
-	: "Ir" (old), "r" (new)
-	: "cc");
-
-	smp_mb();
-	return oldval;
-}
-
-#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
+#if defined(CONFIG_ARM64_LSE_ATOMICS) && defined(CONFIG_AS_LSE)
+#include <asm/atomic_lse.h>
+#else
+#include <asm/atomic_ll_sc.h>
+#endif
 
-static inline int __atomic_add_unless(atomic_t *v, int a, int u)
-{
-	int c, old;
+#undef __ARM64_IN_ATOMIC_IMPL
 
-	c = atomic_read(v);
-	while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c)
-		c = old;
-	return c;
-}
+#include <asm/cmpxchg.h>
 
-#define atomic_inc(v)		atomic_add(1, v)
-#define atomic_dec(v)		atomic_sub(1, v)
+#define ___atomic_add_unless(v, a, u, sfx)				\
+({									\
+	typeof((v)->counter) c, old;					\
+									\
+	c = atomic##sfx##_read(v);					\
+	while (c != (u) &&						\
+	      (old = atomic##sfx##_cmpxchg((v), c, c + (a))) != c)	\
+		c = old;						\
+	c;								\
+ })
 
-#define atomic_inc_and_test(v)	(atomic_add_return(1, v) == 0)
-#define atomic_dec_and_test(v)	(atomic_sub_return(1, v) == 0)
-#define atomic_inc_return(v)    (atomic_add_return(1, v))
-#define atomic_dec_return(v)    (atomic_sub_return(1, v))
-#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
+#define ATOMIC_INIT(i)	{ (i) }
 
-#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)
+#define atomic_read(v)			READ_ONCE((v)->counter)
+#define atomic_set(v, i)		(((v)->counter) = (i))
+#define atomic_xchg(v, new)		xchg(&((v)->counter), (new))
+#define atomic_cmpxchg(v, old, new)	cmpxchg(&((v)->counter), (old), (new))
+
+#define atomic_inc(v)			atomic_add(1, (v))
+#define atomic_dec(v)			atomic_sub(1, (v))
+#define atomic_inc_return(v)		atomic_add_return(1, (v))
+#define atomic_dec_return(v)		atomic_sub_return(1, (v))
+#define atomic_inc_and_test(v)		(atomic_inc_return(v) == 0)
+#define atomic_dec_and_test(v)		(atomic_dec_return(v) == 0)
+#define atomic_sub_and_test(i, v)	(atomic_sub_return((i), (v)) == 0)
+#define atomic_add_negative(i, v)	(atomic_add_return((i), (v)) < 0)
+#define __atomic_add_unless(v, a, u)	___atomic_add_unless(v, a, u,)
+#define atomic_andnot			atomic_andnot
 
 /*
  * 64-bit atomic operations.
  */
-#define ATOMIC64_INIT(i) { (i) }
-
-#define atomic64_read(v)	ACCESS_ONCE((v)->counter)
-#define atomic64_set(v,i)	(((v)->counter) = (i))
-
-#define ATOMIC64_OP(op, asm_op)						\
-static inline void atomic64_##op(long i, atomic64_t *v)			\
-{									\
-	long result;							\
-	unsigned long tmp;						\
-									\